import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
import urllib2
import urllib
import html5lib
import json
import time
import random
from bs4 import BeautifulSoup
outputFile = 'xiaoshuo.txt'
fw = open(outputFile, 'w')
#title='name;phone;address;contect;e-maill;email;QQ;tel;url;\n'
#fw.write(title.encode('gbk'))
headers = {}
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
i=0
for num in range (1,28):
print num
url="http://tieba.baidu.com/p/1627785354?pn="+str(num)
request = urllib2.Request(url=url,headers=headers)
response = urllib2.urlopen(request)
html = response.read()
html = BeautifulSoup(html,"html5lib",from_encoding='utf-8')
cmplist=html.select('.d_post_content')
for index in range(len(cmplist)):
info=cmplist[index].get_text()+"\n"
fw.write(info)
#fw.write(cmplist)
fw.close()
版权声明：本文为liuhongwei_study原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。