進入淘寶網，分別按綜合、銷量排序抓取100頁的所有商品的列表信息。

1、按綜合

import re from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from pyquery import PyQuery as pq #獲取整個網頁的源代碼  from config import * #可引用congif的所有變量 import pymongo import pymysql  # client=pymongo.MongoClient(MONGO_URL) # db = client[MONGO_DB]  # 按綜合排序 100頁   # 打開淘寶鏈接，輸入‘美食'，搜索 # 自動翻頁：先得到總頁數，再轉到 _ 頁，確定 #  # browser = webdriver.PhantomJS(service_args=SERVICE_ARGS) # browser =webdriver.Chrome() browser = webdriver.Firefox() wait = WebDriverWait(browser,10)  def search():  print('正在搜索...')  try:   browser.get('https://www.taobao.com') #用這個網頁'https://s.taobao.com'，無法輸入keywords   input=wait.until(     EC.presence_of_element_located((By.CSS_SELECTOR,'#q')) #打開淘寶，右擊查看元素，定位到搜索框，選擇對應代碼，復制-CSS選擇器，其實就是‘#q'。   )   submit=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'#J_TSearchForm > div.search-button > button')))   input.send_keys(KEYWORD) #模擬操作，輸入內容   submit.click() #點擊提交   total = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > div.total'))) #頁數   return total.text  except TimeoutException :   return search()  # 翻頁 def next_page(page_number):  print('正在翻頁',page_number)  try:   input = wait.until(    # 輸入框    EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input')) # 打開淘寶，右擊查看元素，定位到搜索框，選擇對應代碼，復制-CSS選擇器，其實就是‘#q'。   )   # 搜索按鈕   submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit'))) #未修改   input.clear()   input.send_keys(page_number) # 模擬操作，輸入頁碼   submit.click()   #判斷翻頁是否成功，找到高亮頁碼數，由數子判斷   wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR ,'#mainsrp-pager > div > div > div > ul > li.item.active > span'), str(page_number)))   get_products()  except TimeoutException :   next_page(page_number)  # 解析，獲取每頁的商品并輸出 def get_products():  wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item'))) #加載所有寶貝  html=browser.page_source  doc = pq(html)  items = doc('#mainsrp-itemlist .items .item').items()  for item in items:   product = {    # 'picture':item.find('.pic .img').attr('src'),#用find去獲取內部元素，選擇器是 pic，img，用attr獲取屬性    'image': item.find('.pic .img').attr('data-src'), # 用find去獲取內部元素，選擇器是 pic，img，用attr獲取屬性    'shop_id': item.find('.shop').find('a').attr('data-userid'), # 店鋪 id    'data_id': item.find('.shop').find('a').attr('data-nid'), # 商品 id    'link': item.find('.pic-box-inner').find('.pic').find('a').attr['href'],    'price':item.find('.price').text()[1:-3], # 用text獲取內容    'deal':item.find('.deal-cnt').text()[:-3],    'title':item.find('.title').text().replace(' ',''),    'shop':item.find('.shop').text(),    'location':item.find('.location').text()   }   # print(product)   # print(product['location'])   save_to_mysql(product) ''''' def main():  try:   # search()   total=search() # 此時 total = ‘共 100 頁，'   total=int(re.compile('(/d+)').search(total).group(1)) # 用正則表達式提取數字100   # print(total)   for i in range(2,total+1):    next_page(i)  except Exception:   print('出錯啦')  finally: # 不管有沒有異常，都要執行此操作   browser.close() # 關瀏覽器 '''  def main():  total=search()  total=int(re.compile('(/d+)').search(total).group(1))  for i in range(2,total+1):   next_page(i)#顯示當前爬取網頁的頁數   print ('搞定%d'%i)  def save_to_mysql(product):  # print(product['location'])  #,use_unicode = False  try:   conn = pymysql.connect(host='localhost', user='root', passwd=' ', db='test1', port=3306,charset='utf8' )   cur = conn.cursor() # 創建一個游標對象   sql = """INSERT INTO women_clothes_zonghe VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"""   cur.execute(sql, (product['shop_id'],product['shop'], product['link'],product['data_id'], product['title'], product['price'], product['location'],product['deal'],product['image']))   # cur.execute(sql)   print('- - - - - 數據保存成功 - - - - -')   cur.close()   conn.commit()   conn.close() # 關閉數據  except pymysql.Error as e:   print(e)  if __name__=='__main__':  # 連接數據庫  conn = pymysql.connect(host='localhost', user='root', passwd=' ', db='test1', port=3306,charset="utf8")  cur = conn.cursor() # 創建一個游標對象  cur.execute("DROP TABLE IF EXISTS women_clothes_zonghe") # 如果表存在則刪除  # 創建表sql語句  sqlc = """CREATE TABLE women_clothes_zonghe(   shop_id VARCHAR(500),   shop VARCHAR(500),   link VARCHAR(1000),   data_id varchar(100),   title VARCHAR(1000),   price VARCHAR(500),   location VARCHAR(500),   deal VARCHAR(500),   image VARCHAR(1000)  )"""  cur.execute(sqlc) # 執行創建數據表操作  main()