本文通过七个实战案例演示了 Python 爬虫的核心技术，涵盖正则表达式、XPath、BeautifulSoup、Selenium 及 API 接口分析。内容包含百度贴吧评论抓取、小说章节多线程存储、豆瓣电影榜单解析、电商评论动态加载处理、知乎模拟登录搜索、微博数据获取以及票房数据分析可视化。教程提供完整代码示例，并强调反爬策略应对、数据清洗及合规使用的重要性，适合初学者系统学习网络数据采集流程。
本文通过七个实战案例演示了 Python 爬虫的核心技术，涵盖正则表达式、XPath、BeautifulSoup、Selenium 及 API 接口分析。内容包含百度贴吧评论抓取、小说章节多线程存储、豆瓣电影榜单解析、电商评论动态加载处理、知乎模拟登录搜索、微博数据获取以及票房数据分析可视化。教程提供完整代码示例，并强调反爬策略应对、数据清洗及合规使用的重要性，适合初学者系统学习网络数据采集流程。
在开始之前，请确保已安装以下 Python 库：
pip install requests lxml beautifulsoup4 selenium pymysql pandas matplotlib
若使用 Selenium，还需下载对应浏览器的驱动（如 chromedriver.exe）并配置环境变量。
本案例选取百度贴吧中的帖子，目标是爬取帖子里面的回复内容。涉及知识点：requests 请求发送、re 正则匹配、csv 文件写入。
关键步骤：
源代码：
import csv
import requests
import re
import time
def main(page):
url = f'https://tieba.baidu.com/p/7882177660?pn={page}'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'
}
resp = requests.get(url, headers=headers)
html = resp.text
# 评论内容
comments = re.findall('style="display:;"> (.*?)</div>', html)
# 评论用户
users = re.findall('class="p_author_name j_user_card" href=".*?" target="_blank">(.*?)</a>', html)
# 评论时间
comment_times = re.findall('楼</span><span class="tail-info">(.*?)</span><div', html)
for u, c, t in zip(users, comments, comment_times):
# 筛选数据，过滤掉异常数据
if 'img' in c or 'div' in c or len(u) > 50:
continue
csvwriter.writerow((u, t, c))
print(u, t, c)
print(f'第{page}页爬取完毕')
if __name__ == '__main__':
with open('01.csv', 'a', encoding='utf-8') as f:
csvwriter = csv.writer(f)
csvwriter.writerow(('评论用户', '评论时间', '评论内容'))
for page in range(1, 8): # 爬取前 7 页的内容
main(page)
time.sleep(2)
本案例选取笔趣阁小说网，目标爬取第一篇小说的前 15 个章节内容并存储到 MySQL 数据库。涉及知识点：XPath 解析、多线程并发、MySQL 连接池。
关键步骤：
源代码：
import requests
from lxml import etree
import re
import pymysql
from time import sleep
from concurrent.futures import ThreadPoolExecutor
def get_conn():
conn = pymysql.connect(host="127.0.0.1",
user="root",
password="root",
db="novels",
charset="utf8")
cursor = conn.cursor()
return conn, cursor
def close_conn(conn, cursor):
cursor.close()
conn.close()
def get_xpath_resp(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36'}
resp = requests.get(url, headers=headers)
tree = etree.HTML(resp.text)
return tree, resp
def get_chapters(url):
tree, _ = get_xpath_resp(url)
novel_name = tree.xpath('//*[@id="info"]/h1/text()')[0]
dds = tree.xpath('/html/body/div[4]/dl/dd')
title_list = []
link_list = []
for d in dds[:15]:
title = d.xpath('./a/text()')[0]
title_list.append(title)
link = d.xpath('./a/@href')[0]
chapter_url = url + link
link_list.append(chapter_url)
return title_list, link_list, novel_name
def get_content(novel_name, title, url):
try:
conn, cursor = get_conn()
sql = 'INSERT INTO novel(novel_name,chapter_name,content) VALUES(%s,%s,%s)'
tree, resp = get_xpath_resp(url)
content = re.findall('<div id="content">(.*?)</div>', resp.text)[0]
# 对内容进行清洗
content = content.replace('<br />','\n').replace(' ',' ').replace('全本小说网 www.qb5.tw，最快更新<a href="https://www.qb5.tw/book_116659/">宇宙职业选手</a>最新章节！<br><br>','')
print(title, content)
cursor.execute(sql, [novel_name, title, content])
conn.commit()
except Exception as e:
print(f"Error: {e}")
finally:
sleep(2)
close_conn(conn, cursor)
if __name__ == '__main__':
title_list, link_list, novel_name = get_chapters('https://www.qb5.tw/book_116659/')
with ThreadPoolExecutor(5) as t:
for title, link in zip(title_list, link_list):
t.submit(get_content, novel_name, title, link)
本案例对比两种解析库的使用方式，目标爬取豆瓣电影 Top250 的名称、描述、评分等数据。
关键步骤：
源代码：
import re
from time import sleep
import requests
from lxml import etree
import random
import csv
def main(page, f):
url = f'https://movie.douban.com/top250?start={page*25}&filter='
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.35 Safari/537.36'}
resp = requests.get(url, headers=headers)
tree = etree.HTML(resp.text)
href_list = tree.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div[1]/a/@href')
name_list = tree.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div[2]/div[1]/a/span[1]/text()')
for i, (url_item, name) in enumerate(zip(href_list, name_list)):
f.flush()
try:
get_info(url_item, name)
except:
pass
sleep(1 + random.random())
print(f'第{i+1}页爬取完毕')
def get_info(url, name):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.35 Safari/537.36',
'Host': 'movie.douban.com',
}
resp = requests.get(url, headers=headers)
html = resp.text
tree = etree.HTML(html)
dir = tree.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')[0]
type_ = re.findall(r'property="v:genre">(.*?)</span>', html)
type_ = '/'.join(type_)
country = re.findall(r'地区:</span> (.*?)<br', html)[0]
time_str = tree.xpath('//*[@id="content"]/h1/span[2]/text()')[0]
time_val = time_str[1:5]
rate = tree.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
people = tree.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/div/div[2]/a/span/text()')[0]
print(name, dir, type_, country, time_val, rate, people)
csvwriter.writerow((name, dir, type_, country, time_val, rate, people))
if __name__ == '__main__':
with open('03-movie-xpath.csv', 'a', encoding='utf-8', newline='') as f:
csvwriter = csv.writer(f)
csvwriter.writerow(('电影名称', '导演', '电影类型', '国家', '上映年份', '评分', '评论人数'))
for i in range(10):
main(i, f)
sleep(3 + random.random())
关键步骤：
源代码：
import random
import urllib.request
from bs4 import BeautifulSoup
import codecs
from time import sleep
def main(url, headers):
page = urllib.request.Request(url, headers=headers)
page = urllib.request.urlopen(page)
contents = page.read()
soup = BeautifulSoup(contents, "html.parser")
infofile.write("")
print('爬取豆瓣电影 250: \n')
for tag in soup.find_all(attrs={"class": "item"}):
num = tag.find('em').get_text()
print(num)
infofile.write(num + "\r\n")
name = tag.find_all(attrs={"class": "title"})
zwname = name[0].get_text()
print('[中文名称]', zwname)
infofile.write("[中文名称]" + zwname + "\r\n")
url_movie = tag.find(attrs={"class": "hd"}).a
urls = url_movie.attrs['href']
print('[网页链接]', urls)
infofile.write("[网页链接]" + urls + "\r\n")
info = tag.find(attrs={"class": "star"}).get_text()
info = info.replace('\n', ' ')
info = info.lstrip()
print('[评分评论]', info)
info = tag.find(attrs={"class": "inq"})
if (info):
content = info.get_text()
print('[影评]', content)
infofile.write(u"[影评]" + content + "\r\n")
print('')
if __name__ == '__main__':
infofile = codecs.open("03-movie-bs4.txt", 'a', 'utf-8')
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
i = 0
while i < 10:
print('页码', (i + 1))
num = i * 25
url = 'https://movie.douban.com/top250?start=' + str(num) + '&filter='
main(url, headers)
sleep(5 + random.random())
infofile.write("\r\n\r\n")
i = i + 1
infofile.close()
本案例针对京东商品评论页面，数据为动态加载的 JSON 接口。需要分析开发者工具抓包找到 API 地址。
关键步骤：
源代码：
import requests
import csv
from time import sleep
import random
def main(page, f):
url = 'https://club.jd.com/comment/productPageComments.action'
params = {
'productId': 100011483893,
'score': 0,
'sortType': 5,
'page': page,
'pageSize': 10,
'isShadowSku': 0,
'fold': 1
}
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.35 Safari/537.36',
'referer': 'https://item.jd.com/'
}
resp = requests.get(url, params=params, headers=headers).json()
comments = resp['comments']
for comment in comments:
content = comment['content'].replace('\n', '')
comment_time = comment['creationTime']
score = comment['score']
print(score, comment_time, content)
csvwriter.writerow((score, comment_time, content))
print(f'第{page+1}页爬取完毕')
if __name__ == '__main__':
with open('04.csv', 'a', encoding='utf-8', newline='') as f:
csvwriter = csv.writer(f)
csvwriter.writerow(('评分', '评论时间', '评论内容'))
for page in range(15):
main(page, f)
sleep(5 + random.random())
本案例演示如何使用 Selenium 控制浏览器进行人机验证或扫码登录，随后搜索特定关键词并抓取内容。
关键步骤：
源代码：
from time import sleep
from selenium.webdriver.chrome.service import Service
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.common.by import By
import warnings
def main():
warnings.filterwarnings("ignore")
service = Service('chromedriver.exe')
options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation','enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
driver = Chrome(service=service, options=options)
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": "Object.defineProperty(navigator, 'webdriver', { get: () => false })"
})
driver.get('https://www.zhihu.com/')
print("请在此处完成扫码登录...")
sleep(30)
driver.find_element(By.ID, 'Popover1-toggle').click()
driver.find_element(By.ID, 'Popover1-toggle').send_keys('江汉大学')
sleep(2)
driver.find_element(By.XPATH, '//*[@id="root"]/div/div[2]/header/div[2]/div[1]/div/form/div/div/label/button').click()
driver.implicitly_wait(20)
title = driver.find_element(By.XPATH, '//*[@id="SearchMain"]/div/div/div/div/div[2]/div/div/div/h2/div/a/span').text
driver.find_element(By.XPATH, '//*[@id="SearchMain"]/div/div/div/div/div[2]/div/div/div/div/span/div/button').click()
sleep(2)
content = driver.find_element(By.XPATH, '//*[@id="SearchMain"]/div/div/div/div/div[2]/div/div/div/div/span[1]/div/span/p').text
driver.find_element(By.XPATH, '//*[@id="SearchMain"]/div/div/div/div/div[2]/div/div/div/div/div[3]/div/div/button[1]').click()
sleep(2)
driver.find_element(By.XPATH, '//*[@id="SearchMain"]/div/div/div/div/div[2]/div/div/div/div[2]/div/div/div[2]/div[2]/div/div[3]/button').click()
sleep(2)
divs = driver.find_elements(By.XPATH, '/html/body/div[6]/div/div/div[2]/div/div/div/div[2]/div[3]/div')
try:
for div in divs:
comment = div.find_element(By.XPATH, './div/div/div[2]').text
f.write(comment)
f.write('\n')
print(comment)
except:
driver.close()
if __name__ == '__main__':
with open('05.txt', 'a', encoding='utf-8') as f:
main()
本案例通过微博 API 接口获取指定用户的微博列表，包括转发、评论、点赞数等统计信息。
注意： 代码中 Cookie 为示例，实际使用时需替换为有效 Cookie，且注意隐私合规。
源代码：
import requests
import csv
from time import sleep
import random
def main(page):
url = f'https://weibo.com/ajax/statuses/mymblog?uid=2803301701&page={page}&feature=0&since_id=4824543023860882kp{page}'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
'cookie': 'YOUR_COOKIE_HERE' # 请替换为真实 Cookie
}
resp = requests.get(url, headers=headers)
data_list = resp.json()['data']['list']
for item in data_list:
created_time = item['created_at']
author = item['user']['screen_name']
title = item['text_raw']
reposts_count = item['reposts_count']
comments_count = item['comments_count']
attitudes_count = item['attitudes_count']
csvwriter.writerow((created_time, author, title, reposts_count, comments_count, attitudes_count))
print(created_time, author, title, reposts_count, comments_count, attitudes_count)
print(f'第{page}页爬取完毕')
if __name__ == '__main__':
with open('06-2.csv', 'a', encoding='utf-8', newline='') as f:
csvwriter = csv.writer(f)
csvwriter.writerow(('发布时间', '发布作者', '帖子标题', '转发数', '评论数', '点赞数'))
for page in range(1, 6):
main(page)
sleep(5 + random.random())
本案例选取艺恩娱数票房榜数据，结合 Pandas 和 Matplotlib 进行可视化分析。
关键步骤：
源代码：
import requests
import csv
import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
def main():
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'}
data = {
'r': '0.9936776079863086',
'top': '50',
'type': '0',
}
resp = requests.post('https://ys.endata.cn/enlib-api/api/home/getrank_mainland.do', headers=headers, data=data)
data_list = resp.json()['data']['table0']
for item in data_list:
rank = item['Irank']
MovieName = item['MovieName']
ReleaseTime = item['ReleaseTime']
TotalPrice = item['BoxOffice']
AvgPrice = item['AvgBoxOffice']
AvgAudienceCount = item['AvgAudienceCount']
csvwriter.writerow((rank, MovieName, ReleaseTime, TotalPrice, AvgPrice, AvgAudienceCount))
print(rank, MovieName, ReleaseTime, TotalPrice, AvgPrice, AvgAudienceCount)
def data_analyze():
data = pd.read_csv('07.csv')
data['年份'] = data['上映时间'].apply(lambda x: x.split('-')[0])
df1 = data.groupby('年份')['总票房 (万)'].sum()
plt.figure(figsize=(6, 6))
plt.pie(df1, labels=df1.index.to_list(), autopct='%1.2f%%')
plt.title('各年度上榜电影总票房占比')
plt.show()
df1 = data.groupby('年份')['总票房 (万)'].sum()
plt.figure(figsize=(6, 6))
plt.plot(df1.index.to_list(), df1.values.tolist())
plt.title('各年度上榜电影总票房趋势')
plt.show()
print(data.sort_values(by='平均票价', ascending=False)[['年份', '电影名称', '平均票价']].head(10))
print(data.sort_values(by='平均场次', ascending=False)[['年份', '电影名称', '平均场次']].head(10))
if __name__ == '__main__':
with open('07.csv', 'w', encoding='utf-8', newline='') as f:
csvwriter = csv.writer(f)
csvwriter.writerow(('排名', '电影名称', '上映时间', '总票房 (万)', '平均票价', '平均场次'))
main()
data_analyze()
通过以上七个案例，我们系统地学习了 Python 爬虫的多种应用场景和技术栈。从静态页面的正则/XPath 解析，到动态页面的 Selenium 自动化，再到 API 接口的逆向分析与数据可视化，构成了完整的爬虫技能树。
最佳实践建议：
希望这些案例能帮助初学者快速入门，并在实际项目中灵活应用。
微信公众号「极客日志」，在微信中扫描左侧二维码关注。展示文案：极客日志 zeeklog
使用加密算法（如AES、TripleDES、Rabbit或RC4）加密和解密文本明文。 在线工具，加密/解密文本在线工具，online
解析常见 curl 参数并生成 fetch、axios、PHP curl 或 Python requests 示例代码。 在线工具，curl 转代码在线工具，online
将字符串编码和解码为其 Base64 格式表示形式即可。 在线工具，Base64 字符串编码/解码在线工具，online
将字符串、文件或图像转换为其 Base64 表示形式。 在线工具，Base64 文件转换器在线工具，online
将 Markdown（GFM）转为 HTML 片段，浏览器内 marked 解析；与 HTML转Markdown 互为补充。 在线工具，Markdown转HTML在线工具，online
将 HTML 片段转为 GitHub Flavored Markdown，支持标题、列表、链接、代码块与表格等；浏览器内处理，可链接预填。 在线工具，HTML转Markdown在线工具，online