Python 网络爬虫技术基础与实战指南

3. 代码实现

以下是一个完整的示例，整合了请求、解析、数据存储逻辑。代码中使用了 lxml 进行高效解析，并模拟了数据库插入过程。

import requests
import time
from lxml import etree
import pymysql

# 配置项
BASE_URL = 'https://movie.douban.com/top250?start={}&filter='
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
}
DB_CONFIG = {
    'host': 'localhost',
    'user': 'root',
    'password': 'your_password',
    'database': 'douban',
    'charset': 'utf8mb4'
}

def get_source(url):
    """获取网页源码"""
    try:
        response = requests.get(url, headers=HEADERS, timeout=10)
        response.encoding = 'utf-8'
        if response.status_code == 200:
            return response.text
        else:
            print(f"Request failed with status code: {response.status_code}")
            return None
    except Exception as e:
        print(f"Error fetching URL: {e}")
        return None

def parse_movies(html):
    """解析电影信息"""
    if not html:
        return []
    tree = etree.HTML(html)
    movie_items = tree.xpath("//div[@class='info']")
    result = []
    
    for item in movie_items:
        movie_dict = {}
        # 标题
        title_list = item.xpath("div[@class='hd']/a/span[@class='title']/text()")
        other_title_list = item.xpath("div[@class='hd']/a/span[@class='other']/text()")
        movie_dict['title'] = ''.join(title_list + other_title_list).strip()
        
        # 评分
        star_list = item.xpath("div[@class='bd']/div[@class='star']/span[@class='rating_num']/text()")
        movie_dict['star'] = star_list[0] if star_list else 'N/A'
        
        # 链接
        link_list = item.xpath('div[@class="hd"]/a/@href')
        movie_dict['url'] = link_list[0] if link_list else ''
        
        # 短评
        quote_list = item.xpath("div[@class='bd']/p[@class='quote']/span/text()")
        movie_dict['quote'] = quote_list[0].strip() if quote_list else ''
        
        result.append(movie_dict)
    return result

def save_to_mysql(data_list):
    """保存数据到 MySQL"""
    conn = pymysql.connect(**DB_CONFIG)
    cursor = conn.cursor()
    sql = "INSERT INTO movies (title, star, url, quote) VALUES (%s, %s, %s, %s)"
    try:
        cursor.executemany(sql, data_list)
        conn.commit()
        print(f"Successfully saved {len(data_list)} records.")
    except Exception as e:
        conn.rollback()
        print(f"Database error: {e}")
    finally:
        cursor.close()
        conn.close()

def main():
    all_movies = []
    for i in range(10):  # 爬取前 10 页
        url = BASE_URL.format(i * 25)
        print(f"Fetching page {i+1}...")
        source = get_source(url)
        if source:
            movies = parse_movies(source)
            all_movies.extend(movies)
            time.sleep(1)  # 延时防止被封
    
    if all_movies:
        save_to_mysql(all_movies)

if __name__ == '__main__':
    main()

五、最佳实践与合规性

1. Robots 协议：在爬取前检查网站根目录下的 robots.txt 文件，遵守其规定。
2. 频率限制：合理设置请求间隔，避免对目标服务器造成过大压力。
3. 数据隐私：不抓取个人隐私信息，如手机号、身份证号等。
4. 版权意识：数据仅用于学习研究，不用于非法商用或侵犯版权。

六、总结

掌握 HTTP 原理与 Python 工具是构建爬虫的基础。实际开发中需结合具体场景调整策略，确保稳定高效。通过本教程，读者应能理解从 URL 构造、HTML 解析到数据存储的完整链路，并能编写出具备基本容错能力的爬虫程序。