基本模塊
python爬蟲，web spider。爬取網站獲取網頁數據，并進行分析提取。
基本模塊使用的是 urllib，urllib2，re，等模塊
基本用法，例子：
（1）進行基本GET請求，獲取網頁html
#!coding=utf-8import urllibimport urllib2 url = 'http://www.baidu.com/'# 獲取請求request = urllib2.Request(url)try: # 根據request，得到返回response response = urllib2.urlopen(request)except urllib2.HTTPError, e: if hasattr(e, 'reason'): print e.reason# 讀取response的bodyhtml = response.read()# 讀取response的headersheaders = response.info()
（2）表單提交
#!coding=utf-8import urllib2import urllib post_url = '' post_data = urllib.urlencode({ 'username': 'username', 'password': 'password',}) post_headers = { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:31.0) Gecko/20100101 Firefox/31.0',} request = urllib2.Request( url=post_url, data=post_data, headers=post_headers,) response = urllib2.urlopen(request) html = response.read() (3)
#!coding=utf-8 import urllib2import re page_num = 1url = 'http://tieba.baidu.com/p/3238280985?see_lz=1&pn='+str(page_num)myPage = urllib2.urlopen(url).read().decode('gbk') myRe = re.compile(r'class="d_post_content j_d_post_content ">(.*?)</div>', re.DOTALL)items = myRe.findall(myPage) f = open('baidu.txt', 'a+') import sysreload(sys)sys.setdefaultencoding('utf-8') i = 0texts = []for item in items: i += 1 print i text = item.replace('<br>', '') text.replace('/n', '').replace(' ', '') + '/n' print text f.write(text) f.close() (4)
#coding:utf-8''' 模擬登陸163郵箱并下載郵件內容 '''import urllibimport urllib2import cookielibimport reimport timeimport json class Email163: header = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} user = '' cookie = None sid = None mailBaseUrl='http://twebmail.mail.163.com' def __init__(self): self.cookie = cookielib.CookieJar() cookiePro = urllib2.HTTPCookieProcessor(self.cookie) urllib2.install_opener(urllib2.build_opener(cookiePro)) def login(self,user,pwd): ''' 登錄 ''' postdata = urllib.urlencode({ 'username':user, 'password':pwd, 'type':1 }) #注意版本不同，登錄URL也不同 req = urllib2.Request( url='https://ssl.mail.163.com/entry/coremail/fcg/ntesdoor2?funcid=loginone&language=-1&passtype=1&iframe=1&product=mail163&from=web&df=email163&race=-2_45_-2_hz&module=&uid='+user+'&style=10&net=t&skinid=null', data=postdata, headers=self.header, ) res = str(urllib2.urlopen(req).read()) #print res patt = re.compile('sid=([^"]+)',re.I) patt = patt.search(res) uname = user.split('@')[0] self.user = user if patt: self.sid = patt.group(1).strip() #print self.sid print '%s Login Successful.....'%(uname) else: print '%s Login failed....'%(uname) def getInBox(self): ''' 獲取郵箱列表 ''' print '/nGet mail lists...../n' sid = self.sid url = self.mailBaseUrl+'/jy3/list/list.do?sid='+sid+'&fid=1&fr=folder' res = urllib2.urlopen(url).read() #獲取郵件列表 mailList = [] patt = re.compile('<div/s+class="tdLike Ibx_Td_From"[^>]+>.*?href="([^"]+)"[^>]+>(.*?)<//a>.*?<div/s+class="tdLike Ibx_Td_Subject"[^>]+>.*?href="[^>]+>(.*?)<//a>',re.I|re.S) patt = patt.findall(res) if patt==None: return mailList for i in patt: line = { 'from':i[1].decode('utf8'), 'url':self.mailBaseUrl+i[0], 'subject':i[2].decode('utf8') } mailList.append(line) return mailList def getMailMsg(self,url): ''' 下載郵件內容 ''' content='' print '/n Download.....%s/n'%(url) res = urllib2.urlopen(url).read() patt = re.compile('contentURL:"([^"]+)"',re.I) patt = patt.search(res) if patt==None: return content url = '%s%s'%(self.mailBaseUrl,patt.group(1)) time.sleep(1) res = urllib2.urlopen(url).read() Djson = json.JSONDecoder(encoding='utf8') jsonRes = Djson.decode(res) if 'resultVar' in jsonRes: content = Djson.decode(res)['resultVar'] time.sleep(3) return content ''' Demon'''#初始化mail163 = Email163()#登錄mail163.login('lpe234@163.com','944898186')time.sleep(2) #獲取收件箱elist = mail163.getInBox() #獲取郵件內容for i in elist: print '主題：%s 來自：%s 內容：/n%s'%(i['subject'].encode('utf8'),i['from'].encode('utf8'),mail163.getMailMsg(i['url']).encode('utf8')) （５）需要登陸的情況
#1 cookie的處理 import urllib2, cookielibcookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)urllib2.install_opener(opener)content = urllib2.urlopen('http://XXXX').read() #2 用代理和cookie opener = urllib2.build_opener(proxy_support, cookie_support, urllib2.HTTPHandler) #3 表單的處理 import urllibpostdata=urllib.urlencode({ 'username':'XXXXX', 'password':'XXXXX', 'continueURI':'http://www.verycd.com/', 'fk':fk, 'login_submit':'登錄'}) req = urllib2.Request( url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/', data = postdata)result = urllib2.urlopen(req).read() #4 偽裝成瀏覽器訪問 headers = { 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}req = urllib2.Request( url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/', data = postdata, headers = headers) #5 反”反盜鏈” headers = { 'Referer':'http://www.cnbeta.com/articles'} （６）多線程
from threading import Threadfrom Queue import Queuefrom time import sleep#q是任務隊列#NUM是并發線程總數#JOBS是有多少任務q = Queue()NUM = 2JOBS = 10#具體的處理函數，負責處理單個任務def do_somthing_using(arguments): print arguments#這個是工作進程，負責不斷從隊列取數據并處理def working(): while True: arguments = q.get() do_somthing_using(arguments) sleep(1) q.task_done()#fork NUM個線程等待隊列for i in range(NUM): t = Thread(target=working) t.setDaemon(True) t.start()#把JOBS排入隊列for i in range(JOBS): q.put(i)#等待所有JOBS完成q.join()
scrapy框架
Scrapy框架，Python開發的一個快速,高層次的屏幕抓取和web抓取框架，用于抓取web站點并從頁面中提取結構化的數據。Scrapy用途廣泛，可以用于數據挖掘、監測和自動化測試。
剛開始學習這個框架。不太好評論。只是感覺這個框架有些Java的感覺，需要太多的其他模塊的支持。
（一）創建 scrapy 項目
# 使用 scrapy startproject scrapy_test├── scrapy_test│ ├── scrapy.cfg│ └── scrapy_test│ ├── __init__.py│ ├── items.py│ ├── pipelines.py│ ├── settings.py│ └── spiders│ ├── __init__.py# 進行創建 scrapy 項目
（二）說明
scrapy.cfg: 項目配置文件
items.py: 需要提取的數據結構定義文件
pipelines.py:管道定義，用來對items里面提取的數據做進一步處理，如保存等
settings.py: 爬蟲配置文件
spiders: 放置spider的目錄
（三）依賴包
依賴包比較麻煩。
# python-dev 包的安裝apt-get install python-dev # twisted, w3lib, six, queuelib, cssselect， libxslt pip install w3libpip install twistedpip install lxmlapt-get install libxml2-dev libxslt-dev apt-get install python-lxmlpip install cssselect pip install pyOpenSSL sudo pip install service_identity # 安裝好之后，便可使用 scrapy startproject test 進行創建項目
（四）抓取實例。
（1）創建scrapy項目
dizzy@dizzy-pc:~/Python/spit$ scrapy startproject itzhaopinNew Scrapy project 'itzhaopin' created in: /home/dizzy/Python/spit/itzhaopin You can start your first spider with: cd itzhaopin scrapy genspider example example.comdizzy@dizzy-pc:~/Python/spit$ dizzy@dizzy-pc:~/Python/spit$ cd itzhaopindizzy@dizzy-pc:~/Python/spit/itzhaopin$ tree.├── itzhaopin│ ├── __init__.py│ ├── items.py│ ├── pipelines.py│ ├── settings.py│ └── spiders│ └── __init__.py└── scrapy.cfg # scrapy.cfg: 項http://my.oschina.net/lpe234/admin/new-blog目配置文件# items.py: 需要提取的數據結構定義文件# pipelines.py:管道定義，用來對items里面提取的數據做進一步處理，如保存等# settings.py: 爬蟲配置文件# spiders: 放置spider的目錄
（2）定義要抓取的數據結構 items.py
from scrapy.item import Item, Field# 定義我們要抓取的數據class TencentItem(Item): name = Field() # 職位名稱 catalog = Field() # 職位類別 workLocation = Field() # 工作地點 recruitNumber = Field() # 招聘人數 detailLink = Field() # 職位詳情鏈接 publishTime = Field() # 發布時間
(3)實現Spider類
- Spider是繼承自 scarpy.contrib.spiders.CrawlSpider 的Python類，有3個必須定義的成員。
- name : 名稱，spider的標識。
- start_urls : 一個url列表，spider從這些網頁開始抓取
- parse() : 一個方法。當start_urls里面的網頁抓取下來之后需要調用這個方法來解析網頁內容，同時需要返回下一個需要抓取的網頁，或者返回items列表。
在spiders目錄下面新建一個spider，tencent_spider.py ：
#coding=utf-8 from scrapy.spider import BaseSpider class DmozSpider(BaseSpider): name = 'dmoz' allowed_domains = ['dmoz.org'] start_urls = [ 'http://www.dmoz.org/Computers/Programming/Languages/Python/Books/', 'http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/' ] def parse(self, response): filename = response.url.split('/')[-2] open(filename, 'wb').write(response.info) 這個簡單一些。 使用scrapy crawl dmoz # 即可運行spider
