博客

只爬取一个图集里面所有图片，目标地址url：http://www.mm131.com/xinggan/

用selenium翻页时：

1、图集翻页，启动浏览器获取本页图集的所有，title，url_photo，保存到mongodb，并且递归，直到没有发现最后一页按钮，就跳出结束（翻页时，并没有设置多少页，是一直点击到最后一页，总共大概154页）

2、图片翻页，再次启动一个浏览器，对每次图集翻页获取的url_photo都循环一次获取获取每张图片的url_image（图片翻页时，可以获取到总页数，所有直接循环点击，平均一副图片有30张）

3、对每一个url，获取一次图片，保存到本地

4、注意图片url有防盗链，下载时需要在请求头中写明自己来自哪里。

4、总共 大约154页 * 20个 * 30张 = 图片数量，我没有跑完，你可以试试。

5、 selenium3.x已经放弃PhantomJS了，这里运行会出现警告，不过没关系，还是可以运行代码，也可以把这部分修改下。

所有代码如下：

import pymongo
from requests import RequestException
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from config import *
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
from selenium.common.exceptions import TimeoutException
import os
from hashlib import md5
import requests
'''
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
browser = webdriver.Chrome(chrome_options=chrome_options)
'''
browser = webdriver.PhantomJS(service_args=SERVICE_ARGS) 
browser.set_window_size(1400,900)  
tar_url ='http://www.mm131.com/xinggan/'
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
browser_1.get(tar_url)
wait = WebDriverWait(browser_1, 10)

#图集翻页
def next_to_page():
    try:
        print('图集下一页')
        e1 = browser_1.find_element_by_link_text('下一页')  
        if e1.is_displayed():
            nextpage =  wait.until(EC.element_to_be_clickabl((By.LINK_TEXT, '下一页')))
            nextpage.click()         wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.page_now')))
            url_photo=get_photo_url() 
            number=get_page_number(url_photo) 
            next_to_photo(url_photo,number)     
            next_to_page()  #递归           
        else:
            print('所有图集翻页完毕')
            browser.close()
    except TimeoutException:
        next_to_page()

#图集这一页获取url
def get_photo_url():
    html = browser.page_source
    doc = pq(html)
    items = doc('.list-left.public-box>dd').items()
    for item in items:
        url = item.find('a').attr('href')  
        try:
            resonpse = requests.get(url)
            if resonpse.status_code == 200: 
                dic = {
                    'title': item.text(),
                    'url_photo': item.find('a').attr('href')}
                save_to_mongo(dic)
                #title = dic.get('title')
                url_photo = dic.get('url_photo')
                return url_photo
        except RequestException:
            pass

# 获取每一页的页数
def get_page_number(url_photo):  
    try:
        doc = pq(url=url_photo, encoding='gbk')
        page = doc.find('.page-ch').text()
        number = int(page[1:3])
        print('图片页数是:',number)
        return number
    except Exception:
        pass


def next_to_photo(url_photo,number):
    print('正在获取图片所有链接')
    browser_2 = webdriver.PhantomJS(service_args=SERVICE_ARGS) 
    browser_2.set_window_size(1400, 900)
    wait = WebDriverWait(browser, 10)
    browser_2.get(url_photo)
    try:
        for i in range(1,number):  
            if i != number-1
                nextpage = wait.until(EC.element_to_be_clickable((By.LINK_TEXT, '下一页')))
                html = browser.page_source  
                doc = pq(html)
                imageUrl = doc.find('.content-pic>a>img').attr('src')  
                print('-------------------------------------')
                download_image(imageUrl)
                #print(imageUrl)
                nextpage.click()
                wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'.page_now')))
                html = browser.page_source
            else:
                browser.close()
    except TimeoutException:
        next_to_photo(number)

# 把url传过来，用浏览器去请求图片对应的url
def download_image(imageUrl):  
    print('下载成功', imageUrl)
    Picreferer = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/63.0.3239.108 Safari/537.36', 'Referer': 'http://www.mm131.com/xinggan/4210.html'}
    resonpse = requests.get(imageUrl, headers=Picreferer) 

    if resonpse.status_code == 200:
        save_image(resonpse.content)  

def save_image(content):
    filename = 'E:\\PycharmProjects\\Crawler\\MM131\\photo'
    file_path = '{0}/{1}.{2}'.format(filename, md5(content).hexdigest(), 'jpg')  
    if not os.path.exists(file_path):  
        with open(file_path, 'wb') as f: 
            f.write(content)
            f.close()
            
def save_to_mongo(result): 
    try:
        if db[MONGO_TABLE].insert_one(result):
            print('存到mongodb成功',result)
    except Exception:
        print('存到mongodb失败',result)

if __name__ == '__main__':
    next_to_page()

SERVICE_ARGS = ['--load-images=false','--disk-cache=true']
MONGO_URL = 'localhost'
MONGO_DB = '图片'
MONGO_TABLE = '图片'