最近剛學習Python，做了個簡單的爬蟲，作為一個簡單的demo希望幫助和我一樣的初學者。

代碼使用python2.7做的爬蟲  抓取51job上面的職位名，公司名，薪資，發(fā)布時間等等。

直接上代碼，代碼中注釋還算比較清楚 ，沒有安裝mysql需要屏蔽掉相關代碼：

#!/usr/bin/python # -*- coding: UTF-8 -*-  from bs4 import BeautifulSoup import urllib import urllib2 import codecs import re import time import logging import MySQLdb   class Jobs(object):    # 初始化   """docstring for Jobs"""    def __init__(self):     super(Jobs, self).__init__()          logging.basicConfig(level=logging.DEBUG,          format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')     #數(shù)據(jù)庫的操作，沒有mysql可以做屏蔽     self.db = MySQLdb.connect('127.0.0.1','root','rootroot','MySQL_Test',charset='utf8')     self.cursor = self.db.cursor()      #log日志的顯示     self.logger = logging.getLogger("sjk")      self.logger.setLevel(level=logging.DEBUG)      formatter = logging.Formatter(       '%(asctime)s - %(name)s - %(levelname)s - %(message)s')     handler = logging.FileHandler('log.txt')     handler.setFormatter(formatter)     handler.setLevel(logging.DEBUG)     self.logger.addHandler(handler)      self.logger.info('初始化完成')    # 模擬請求數(shù)據(jù)   def jobshtml(self, key, page='1'):     try:       self.logger.info('開始請求第' + page + '頁')       #網(wǎng)頁url       searchurl = "https://search.51job.com/list/040000,000000,0000,00,9,99,{key},2,{page}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="        user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:59.0) Gecko/20100101 Firefox/59.0'       #設置請求頭       header = {'User-Agent': user_agent, 'Host': 'search.51job.com',            'Referer': 'https://www.51job.com/'}       #拼接url       finalUrl = searchurl.format(key=key, page=page)              request = urllib2.Request(finalUrl, headers=header)        response = urllib2.urlopen(request)       #等待網(wǎng)頁加載完成       time.sleep(3)       #gbk格式解碼       info = response.read().decode('gbk')        self.logger.info('請求網(wǎng)頁網(wǎng)頁')        self.decodeHtml(info=info, key=key, page=page)      except urllib2.HTTPError as e:       print e.reason    # 解析網(wǎng)頁數(shù)據(jù)   def decodeHtml(self, info, key, page):     self.logger.info('開始解析網(wǎng)頁數(shù)據(jù)')     #BeautifulSoup 解析網(wǎng)頁     soup = BeautifulSoup(info, 'html.parser')     #找到class = t1 t2 t3 t4 t5 的標簽數(shù)據(jù)     ps = soup.find_all(attrs={"class": re.compile(r'^t[1-5].*')})     #打開txt文件 a+ 代表追加     f = codecs.open(key + '.txt', 'a+', 'UTF-8')     #清除之前的數(shù)據(jù)信息     f.truncate()      f.write('/n------------' + page + '--------------/n')      count = 1      arr = []     #做一些字符串的處理，形成數(shù)據(jù)格式  iOS開發(fā)工程師 有限公司 深圳-南山區(qū) 0.9-1.6萬/月 05-16     for pi in ps:       spe = " "       finalstr = pi.getText().strip()       arr.append(finalstr)       if count % 5 == 0:         #每一條數(shù)據(jù)插入數(shù)據(jù)庫，如果沒有安裝mysql 可以將當前行注釋掉         self.connectMySQL(arr=arr)         arr = []         spe = "/n"       writestr = finalstr + spe       count += 1       f.write(writestr)     f.close()          self.logger.info('解析完成')  #數(shù)據(jù)庫操作 沒有安裝mysql 可以屏蔽掉   def connectMySQL(self,arr):     work=arr[0]     company=arr[1]     place=arr[2]     salary=arr[3]     time=arr[4]      query = "select * from Jobs_tab where /     company_name='%s' and work_name='%s' and work_place='%s' /     and salary='%s' and time='%s'" %(company,work,place,salary,time)     self.cursor.execute(query)      queryresult = self.cursor.fetchall()     #數(shù)據(jù)庫中不存在就插入數(shù)據(jù) 存在就可以更新數(shù)據(jù) 不過我這邊沒有寫     if len(queryresult) > 0:       sql = "insert into Jobs_tab(work_name,company_name,work_place,salary/           ,time) values('%s','%s','%s','%s','%s')" %(work,company,place,salary,time)              try:         self.cursor.execute(sql)         self.db.commit()                except Exception as e:         self.logger.info('寫入數(shù)據(jù)庫失敗')         #模擬登陸   # def login(self):   #   data = {'action':'save','isread':'on','loginname':'18086514327','password':'kui4131sjk'}     # 開始抓取 主函數(shù)   def run(self, key):      # 只要前5頁的數(shù)據(jù) key代表搜索工做類型 這邊我是用的ios page是頁數(shù)     for x in xrange(1, 6):       self.jobshtml(key=key, page=str(x))      self.logger.info('寫入數(shù)據(jù)庫完成')      self.db.close()  if __name__ == '__main__':    Jobs().run(key='iOS')