RSS Site Crawler

import scrapy
import datetime, time


class PersonalAssetsSpider(scrapy.Spider):
	name = "PersonalAssets"
	#allowed_domains = ['https://patplc.co.uk']
	start_urls = ['http://127.0.0.1/test.html', ]

	# Custom settings that I can read in the pipeline & put in feed.
	custom_settings = {}
	custom_settings['RSS_TITLE'] = 'Personal Assets Trust'
	custom_settings['RSS_LINK'] = start_urls[0]
	custom_settings['RSS_OUTPUT_FILE'] = 'PersonalAssets.rss'


	def parse(self, response):
		"""Select page elements to pick for the generated xml element"""

		# Best Methed:
		# Curl page onto localhost for dev
		# run 'scrapy shell http://127.0.0.1/test.html'
		# Can then experiment: response.css('li.views-row a::attr(title)') 
		# dot to specify class.  Space separated sub elements
		for viewRow in response.css('li.views-row'):

			# Note the guid is a unique descriptor, just repeat url
			# I've set the order here in FEED_EXPORT_FIELDS cfg variable
			title = viewRow.css('a::attr("title")').extract_first()		
			link = viewRow.css('a::attr("href")').extract_first()

			yield {
			'title': title,
			'link': link,
			'guid' : link,
			'description': "Personal Assets "+title,
			}

from scrapy.exporters import XmlItemExporter 

# Create subclas of XmlItemExporter Override the default XmlItemExporter and update cfg FEED_EXPORTERS to use this instead
class RssXmlItemExporter(XmlItemExporter):

	# Apparently not essential to define, as takes from parent class
	# Doing so just in case I want to add more into to it
	def __init__(self, file, **kwargs):
		XmlItemExporter.__init__(self, file, **kwargs)

	def start_exporting_rss(self, rss_title, rss_link):
		"""
		Adds opening tags, including channel and rss
		Additionall adds a title and link to the feed.
		"""
		self.xg.startDocument()

		# IRB MYsuff
		self.xg.startElement("rss", {'version':'2.0'})
		self._beautify_newline(new_item=True)
		self.xg.startElement("channel", {})
		self._beautify_newline(new_item=True)
		self._export_xml_field('title',rss_title,1)
		self._export_xml_field('link',rss_link,1)
		# End Mysuff
		
		self.xg.startElement(self.root_element, {})
		self._beautify_newline(new_item=True)



	def finish_exporting_rss(self):
		"""Closes off my custom rss and channel tags"""
		self.xg.endElement(self.root_element)

		# IRB MYsuff
		self._beautify_newline(new_item=True)
		self.xg.endElement("channel")
		self._beautify_newline(new_item=True)
		self.xg.endElement("rss")
		# End Mysuff

		self.xg.endDocument()

FEED_EXPORTERS = {'xml' : 'SiteCrawler.feedexport.RssXmlItemExporter'}

from SiteCrawler.feedexport import RssXmlItemExporter 

class SitecrawlerPipeline(object):

	def __init__(self, settings):

		# Following items were set in spider using custom_settings dict.
		self.rss_title = settings.get("RSS_TITLE")
		self.rss_link = settings.get("RSS_LINK")
		self.rss_output_file = "output_feeds/" + settings.get("RSS_OUTPUT_FILE")

		# Settings for the exporter.  These come from global settings.py
		exporterSettings = 	{}
		exporterSettings['fields_to_export'] = settings.get("FEED_EXPORT_FIELDS")
		exporterSettings['indent'] = settings.get("FEED_EXPORT_INDENT")
		exporterSettings['encoding'] = 'utf-8'
		#exporterSettings['export_empty_fields'] = False 
		#exporterSettings['item_element'] = 
		#exporterSettings['root_element'] = 
		
		# Open the rss file for writing 
		self.file = open(self.rss_output_file,"wb")	# Overwrites existing

		# Now setup the exporter to be used.  
		# Can pass params separately, but I'm using dict.
		self.exporter = RssXmlItemExporter(self.file, **exporterSettings)

		# Useful to know how to tie signals to functions.  
		# open/close_spider automatically setup
		# dispatcher.connect(self.SpiderOpenFunc, signals.spider_opened)
		# dispatcher.connect(self.SpiderCloseFunc, signals.spider_closed)


	@classmethod
	def from_crawler(cls, crawler):
		"""
		This function changes how the crawler engine calls the pipeline 
		__init__ function.  The additional parameters are added.  Therefore 
		need to update init above to include extra param(s). This allows 
		access to global settings (settings.py) and custom setting from within 
		spider
		"""
		settings = crawler.settings
		return cls(settings)
	
	def open_spider(self, spider):
		"""Use custom functions I defined that add rss and channel tags"""
		self.exporter.start_exporting_rss(self.rss_title, self.rss_link);

	def close_spider(self, spider):
		"""USe custom function that closes the new rss and channel tags"""
		self.exporter.finish_exporting_rss();
		self.file.close()

	def process_item(self, item, spider):
		"""Unchanged to what was here before"""
		self.exporter.export_item(item)
		return item

ITEM_PIPELINES = {
    'SiteCrawler.pipelines.SitecrawlerPipeline': 300,
}