#!/usr/bin/env python3
"""
Extract, transform and load a list of RSS feeds into BigQuery.
To execute the ETL process, run:
    rss = rssLoad()
    rss.rss_list_to_df()
    rss.rss_to_bigquery()
"""

DEFAULT_PROJECT_ID = 'allyd-data'
DEFAULT_DATASET_ID = 'rss_data'
DEFAULT_TABLE_ID = 'daily_feeds'
DEFAULT_RSS_INPUT = '/home/gordon_silvera/allyd-data/notebooks/etl_inputs.yaml'
DEFAULT_RSS_KEY = 'rss_feeds'
SYS_PATH = '/home/gordon_silvera/allyd-data/src'
CREDS_PATH = '/home/gordon_silvera/creds.json'

import sys
sys.path.append(SYS_PATH)


class rssInputs():
    """Provide inputs for RSS ETL process. 
    """
    def __init__(self):
        self.yaml_input_file = DEFAULT_RSS_INPUT
        self.yaml_input_key = DEFAULT_RSS_KEY
        self.rss_list = self.get_rss_list()

    def get_rss_list(self):
        from data_mgmt import yaml_data
        inputs = yaml_data.get_yaml_data(self.yaml_input_file)
        self.rss_list = inputs[self.yaml_input_key]
        return self.rss_list


class rssData():
    """Transform RSS URL into a pandas dataframe.
    """
    def __init__(self, rss_url):
        self.rss_url = rss_url
    
    def _get_rss_data(self):
        """Output RSS dictionary from URL"""
        import feedparser
        d = feedparser.parse(self.rss_url)
        return d
    
    def _get_entry_list(self):
        """Get RSS entries from RSS dictionary"""
        d = self._get_rss_data()
        return d['entries']

    def _get_links(self, entry):
        """Get article links"""
        link_dict = dict()
        for item in entry['links']:
            if 'image' in item['type']:
                link_dict.update({'image_url' : item['href']})
            else:
                link_dict.update({'image_url' : ''})
            if 'text' in item['type']:
                link_dict.update({'text_url' : item['href']})
            else:
                link_dict.update({'text_url' : ''})
        return link_dict

    def _get_content(self):
        entry = self._get_entry_list()
        content_dict = dict()
        for item in entry['links']:
            if 'image' in item['type']:
                content_dict.update({'image_url' : item['href']})
            else:
                content_dict.update({'image_url' : ''})
            if 'text' in item['type']:
                content_dict.update({'content_text' : item['href']})
            else:
                content_dict.update({'content_text' : ''})
        return content_dict

    def _get_entry_value(self, obj, key):
        if isinstance(key, str):
            try:
                return obj[key]
            except KeyError:
                return ''
        elif isinstance(key, list):
            o = obj
            for k in key:
                try:
                    o = o[k]
                except:
                    return ''
            return o
    
    def rss_to_df(self):
        """For a single RSS URL, return RSS entries into a dataframe.
        """
        import pandas as pd
        
        # get rss entries and feed data
        entry_list = self._get_entry_list()
        feed = self._get_rss_data()['feed']
        entry_df = pd.DataFrame()
        
        # loop through entries
        for entry in entry_list:
            entry_dict = dict(
                entry_id = [self._get_entry_value(entry, 'id')],
                article_title = [self._get_entry_value(entry, 'title')],
                article_title_value = [self._get_entry_value(entry, ['title_detail', 'value'])],
                source_url = [self._get_entry_value(entry, ['title_detail', 'base'])],
                source_title = [self._get_entry_value(feed, 'title')],
                source_updated = [self._get_entry_value(feed, 'updated')],
                source_language = [self._get_entry_value(feed, 'language')],
                source_update_period = [self._get_entry_value(feed, 'sy_updateperiod')],
                source_update_frequency = [self._get_entry_value(feed, 'sy_updatefrequency')],
                article_text_url = [self._get_links(entry)['text_url']],
                article_image_url = [self._get_links(entry)['image_url']],
                article_link = [self._get_entry_value(entry, 'link')],
                article_published = [self._get_entry_value(entry, 'published')],
                article_published_parsed = [self._get_entry_value(entry, 'published_parsed')],
                article_guidislink = [self._get_entry_value(entry, 'guidislink')],
                article_summary = [self._get_entry_value(entry, 'summary')],
                article_content_url = [self._get_entry_value(entry, ['content', 0, 'base'])],
                article_content_value = [self._get_entry_value(entry, ['content', 0, 'value'])],
            )
            entry_tmp = pd.DataFrame.from_dict(entry_dict)
            entry_df = entry_df.append(entry_tmp)
        self.rss_df = entry_df.drop_duplicates()
        return self.rss_df


class rssLoad():
    """ETL process to load RSS data into BigQuery. Execute the following
    commands to execute ETL process:
        rss = rssLoad()
        rss.rss_list_to_df()
        rss.rss_to_bigquery()
    """
  
    def __init__(self):
        self.rss_inputs = rssInputs()
        self.rss_list = self.rss_inputs.rss_list
        self.project_id = DEFAULT_PROJECT_ID
        self.dataset_id = DEFAULT_DATASET_ID
        self.table_id = DEFAULT_TABLE_ID 
        self.credentials_path = CREDS_PATH
        
    def rss_list_to_df(self):
        """Returns a list of RSS feeds to a dataframe. Add name of YAML input
        file to the default parameters. 
        """
        import pandas as pd
        df = pd.DataFrame()
        for rss in self.rss_list:
            print('RUNNING: ' + rss)
            rss_data = rssData(rss)
            df_tmp = rss_data.rss_to_df()
            df = df.append(df_tmp)
        self.df = df.drop_duplicates()
        print('RSS dataframe has been added to `self.df`.')

    def rss_to_bigquery(self, if_exists='replace'):
        """Write RSS output to BigQuery. Output table is date partitioned. 
        
        Parameters
        ----------
        if_exists:  (optional) string, how to treat BQ load if the 
                    table already exists. Either 'replace', 'append',
                    or 'fail'. Default = 'replace'.
        """
        from google.oauth2 import service_account
        import pandas_gbq
        import datetime as dt
        # check BigQuery/Service Account credentials
        if not self.credentials_path:
            self.credentials_path = input(
                "Upload your GCP Service Account credentials and enter the path below:\n")
            credentials = service_account.Credentials.from_service_account_file(self.credentials_path) 
        else:
            credentials = service_account.Credentials.from_service_account_file(self.credentials_path) 
        # check for inputs
        if None in (self.project_id, self.dataset_id, self.table_id):
            raise Exception('You must specify the output project_id, dataset_id and table_id in rssOutputs().')
        
        self.df.to_gbq(
            '{}.{}_{}'.format(
                self.dataset_id, 
                self.table_id, 
                dt.datetime.now().strftime("%Y%m%d")
            ),
            self.project_id, 
            if_exists=if_exists,
            credentials=credentials
        )
        print('RSS has been loaded to BigQuery: `{}.{}_{}`.'.format(
            self.dataset_id, 
            self.table_id, 
            dt.datetime.now().strftime("%Y%m%d")
        ))
        print('RSS ETL is complete!')

