【爬虫】某阁小说搜索爬取
效果图接下来是源码使用了第三方库：requests和lxml安装方法自行百度即可。不喜欢安装的话文末有我打包的exe程序，自行下载。
import time
import requests
from lxml import etree
class MyBook():
def __init__(self):
self.index = 1
self.headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36'
}
def get_book_url(self,bookname):
url = 'http://www.xbiquge.la/modules/article/waps.php'
data = {
'searchkey': bookname
}
res = requests.post(url,data=data).content.decode('utf-8')
search_ele = etree.HTML(res)
book_ele = search_ele.xpath('//table[@class="grid"]/tr')
if book_ele:
num = 0
print('查询结果：')
for i in book_ele:
book_name = ''.join(i.xpath('td//text()'))
author = i.xpath('td/text()')
num += 1
print(f'{num}.书名:{book_name} , 作者：{author}')
choice = input('请输入编号，进行下载：')
mychoice = book_ele
book_link = mychoice.xpath('td/a/@href')
book_name = ''.join(mychoice.xpath('td//text()'))
self.get_cha_url(book_link,book_name)
else:
print('没有搜索到内容，可搜书名和作者，请您少字也别输错字！')
def get_cha_url(self,book_link,book_name):
print("下载中...")
res = requests.get(book_link, headers=self.headers).content.decode('utf-8')
ele = etree.HTML(res)
char_list = ele.xpath('//div[@id="list"]/dl/dd/a/@href')
char_list = ['http://www.xbiquge.la' + i for i in char_list]
for char_url in char_list:
self.get_content(char_url,book_name)
print('恭喜，下载完毕')
def get_content(self,char_url,book_name):
if self.index == 3:
self.index = 1
return
try:
cha_res = requests.get(char_url, headers=self.headers).content.decode('utf-8')
except:
print(f"当前章节出错，章节url:{char_url},重新获取中")
self.index += 1
time.sleep(3)
self.get_content(char_url, book_name)
else:
cha_ele = etree.HTML(cha_res)
cha_name = cha_ele.xpath('//h1/text()')
char_content = cha_ele.xpath('//div[@id="content"]/text()')
print('正在存储：', cha_name)
with open(f'{book_name}.txt', 'a', encoding='utf-8') as w:
w.write(cha_name + '\n')
for i in char_content:
w.write(i[:50])
if i:
w.write('\n')
if 0 < len(i) < 50:
w.write(i)
elif 50 <= len(i) < 100:
w.write(i)
w.write('\n')
w.write(i)
elif 100 <= len(i) < 150:
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
elif 150 <= len(i) < 200:
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
elif 200 <= len(i) < 250:
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
elif 250 <= len(i):
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
w.write('\n')
w.write(i)
else:
w.write(i)
w.write('\n')
bookname = input('请输入书名，输入完按回车：')
mybook = MyBook()
mybook.get_book_url(bookname)
成品地址：https://wwi.lanzoux.com/inGmMok5o6j
谢谢@Thanks！
感谢分享，谢谢提供分享 感谢楼主分享！大牛有你更精彩！
支持楼主，谢谢分享。 谢谢分享
页:
[1]