from __future__ import absolute_import
import scrapy, re, pandas as pd
from detikcom.items import DetikcomItem



class DetikSpider(scrapy.Spider):
    name = 'detikcom2'
    allowed_domain = 'www.detik.com'
    start_urls = ['https://www.detik.com/']
    # https://www.detik.com/search/searchall?query=banjir%20jakarta&siteid=2&sortby=time&fromdatex=01/01/2016&page=2
    data = pd.read_csv('D:/Kuliah/Skripsi/Scraping/selenium/selenium_detik_link.csv')
    
    def start_requests(self):
        header = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
        urls = self.data['link']
        for url in urls:
            yield scrapy.http.Request(url+'?single=1', callback = self.get_content, headers = header)
    
    
    def get_content(self, response):
        item = DetikcomItem()
        text = response.xpath('//div[@class="detail__body-text itp_bodycontent"]/p//text()').getall()
        if len(text)==0:
            text = response.xpath('//div[@class="detail__body-text itp_bodycontent"]/text()').getall()
        #d = self.data[self.data['link']==response.url]
        item['content'] = ' '.join([t.strip() for t in text])
        #item['date'] = d['date']
        #item['title'] = d['title']
        item['link'] = response.url
        item['tags'] = ', '.join(response.xpath("//a[@class='nav__item']/text()").getall())
        yield item