基于scrapy可见可得的爬虫工具arachnado

class ArachnadoSpider(scrapy.Spider): """ A base spider that contains common attributes and utilities for all Arachnado spiders """ crawl_id = None domain = None motor_job_id = None def __init__(self, *args, **kwargs): super(ArachnadoSpider, self).__init__(*args, **kwargs) # don't log scraped items logging.getLogger("scrapy.core.scraper").setLevel(logging.INFO) def get_page_item(self, response, type_='page'): return { 'crawled_at': datetime.datetime.utcnow(), 'url': response.url, 'status': response.status, 'headers': response.headers, 'body': response.body_as_unicode(), 'meta': response.meta, '_type': type_, } class CrawlWebsiteSpider(ArachnadoSpider): """ A spider which crawls all the website. To run it, set its ``crawl_id`` and ``domain`` arguments. """ name = 'crawlwebsite' custom_settings = { 'DEPTH_LIMIT': 10, } def __init__(self, *args, **kwargs): super(CrawlWebsiteSpider, self).__init__(*args, **kwargs) self.start_url = add_scheme_if_missing(self.domain) def start_requests(self): self.logger.info("Started job %s#%d for domain %s", self.motor_job_id, self.crawl_id, self.domain) yield scrapy.Request(self.start_url, self.parse_first, dont_filter=True) def parse_first(self, response): # If there is a redirect in the first request, use the target domain # to restrict crawl instead of the original. self.domain = get_netloc(response.url) self.crawler.stats.set_value('arachnado/start_url', self.start_url) self.crawler.stats.set_value('arachnado/domain', self.domain) allow_domain = self.domain if self.domain.startswith("www."): allow_domain = allow_domain[len("www."):] self.get_links = LinkExtractor( allow_domains=[allow_domain] ).extract_links for elem in self.parse(response): yield elem def parse(self, response): if not isinstance(response, HtmlResponse): self.logger.info("non-HTML response is skipped: %s" % response.url) return yield self.get_page_item(response) for link in self.get_links(response): yield scrapy.Request(link.url, self.parse)