import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule from wxapp.items import WxappItem class WxSpider(CrawlSpider):
name = 'wx'
allowed_domains = ['wxapp-union.com']
start_urls = ['http://www.wxapp-union.com/portal.php?mod=list&catid=2&page=1'] rules = (
Rule(LinkExtractor(allow=r'.*mod=list&catid=2&page=\d+'), follow=True),
Rule(LinkExtractor(allow=r'.*article-.+\.html'), callback='parse_detail', follow=False),
) def parse_detail(self, response):
detail_href = response.request.url
title = response.xpath('//h1[@class="ph"]/text()').get()
content = response.xpath('//td[@id="article_content"]//text()').getall()
content = [c.strip() for c in content]
content = ''.join(content).strip()
pub_time = response.xpath('//p[@class="authors"]/span/text()').get()
author = response.xpath('//p[@class="authors"]/a/text()').get()
item = WxappItem(title=title, content=content, detail_href=detail_href, pub_time=pub_time, author=author)
yield item
from scrapy.exporters import JsonLinesItemExporter, JsonItemExporter class WxappPipeline(object):
def __init__(self):
"""
爬虫开始的时候执行
"""
self.fp = open("data.json", 'wb')
self.exporter = JsonItemExporter(self.fp, ensure_ascii=False, encoding='utf-8') def open_spider(self, spider):
"""
爬虫开始的时候执行
:param spider:
:return:
"""
pass def process_item(self, item, spider):
self.exporter.export_item(item)
return item def close_spider(self, spider):
"""
爬虫结束的时候执行
:param spider:
:return:
"""
self.fp.close()
import scrapy class WxappItem(scrapy.Item):
title = scrapy.Field()
content = scrapy.Field()
pub_time = scrapy.Field()
author = scrapy.Field()
detail_href = scrapy.Field()
