分析

使用CrawlSpider結合LinkExtractor和Rule爬取網(wǎng)頁信息

LinkExtractor用于定義鏈接提取規(guī)則，一般使用allow參數(shù)即可

LinkExtractor(allow=(), # 使用正則定義提取規(guī)則       deny=(), # 排除規(guī)則       allow_domains=(), # 限定域名范圍       deny_domains=(), # 排除域名范圍       restrict_xpaths=(), # 使用xpath定義提取隊則       tags=('a', 'area'),        attrs=('href',),        canonicalize=False,       unique=True,        process_value=None,       deny_extensions=None,        restrict_css=(), # 使用css選擇器定義提取規(guī)則       strip=True):

Rule用于定義CrawlSpider的爬取規(guī)則，由Spider內部自動識別，提交請求、獲取響應，交給callback指定的回調方法處理response

如果指定了callback，參數(shù)follow默認為False；如果callback為None，follow默認為True

Rule(link_extractor, # LinkExtractor對象,必選參數(shù)   callback=None, # 回調方法,可選   cb_kwargs=None,    follow=None, # 是否進行深度爬取,True、False   process_links=None, # 用于處理鏈接(有些反爬策略是返回假的url)   process_request=identity)

源碼

items.py

class BosszhipinItem(scrapy.Item):  """Boss直聘Pytho職位爬蟲Item"""  # 職位名稱  position=scrapy.Field()  # 公司名稱  company=scrapy.Field()  # 薪資  salary=scrapy.Field()  # 工作地點  location=scrapy.Field()  # 學歷要求  education=scrapy.Field()  # 工作時間  year=scrapy.Field()

spiders/bosszhipin_spider.py

# !/usr/bin/env python# -*- coding:utf-8 -*-import scrapyfrom scrapy.spider import CrawlSpider,Rulefrom scrapy.linkextractors import LinkExtractorfrom myscrapy.items import BosszhipinItemclass BosszhipinSpider(CrawlSpider):  """  Boss直聘Python職位爬蟲Spider    使用CrawlSpider基類實現(xiàn)  """  name = 'bosszhipin'  allowed_domains=['zhipin.com',]  start_urls=['http://www.zhipin.com/c100010000/h_100010000/?query=Python&page=1',]  # 鏈接提取器對象(規(guī)定鏈接提取規(guī)則)  link_extractor=LinkExtractor(allow=(r'page=/d+'))  # 鏈接提取規(guī)則對象列表  # 自動調用callback指定的方法,去取爬取由link_extractor指定的鏈接提取規(guī)則匹配到的url  # 原理:link_extractor.extract_links(response)返回匹配到的鏈接  rules = [    Rule(link_extractor=link_extractor,callback='parse_page',follow=True),  ]  def parse_page(self,response):    """定義回調方法,用于解析每個response對象"""    job_list=response.xpath('//div[@class="job-list"]//li')    for job in job_list:      position = job.xpath('.//div[@class="info-primary"]//h3[@class="name"]/a/text()')[0].extract()      salary =job.xpath('.//div[@class="info-primary"]//h3[@class="name"]//span/text()')[0].extract()      company =job.xpath('.//div[@class="company-text"]//a/text()')[0].extract()      location =job.xpath('.//div[@class="info-primary"]/p/text()[1]')[0].extract()      year =job.xpath('.//div[@class="info-primary"]/p/text()[2]')[0].extract()      education =job.xpath('.//div[@class="info-primary"]/p/text()[3]')[0].extract()      item=BosszhipinItem()      item['position']=position      item['salary']=salary      item['company']=company      item['location']=location      item['year']=year      item['education']=education      yield item