Scrapy 框架配置免费代理 IP 及爬虫防封方法 | 极客日志

Scrapy 框架配置免费代理 IP 及爬虫防封方法 | 极客日志

import requests
import time

def check_proxy(proxy):
    """
    检测代理是否可用
    :param proxy: 代理格式 'http://ip:port'
    :return: True/False
    """
    test_url = "http://httpbin.org/ip" # 验证 IP 的测试地址
    timeout = 5 # 超时时间 5 秒
    try:
        response = requests.get(
            test_url, proxies={"http": proxy, "https": proxy}, timeout=timeout
        )
        if response.status_code == 200:
            print(f"代理可用：{proxy}")
            return True
    except Exception as e:
        # 捕获连接超时、拒绝连接等异常
        print(f"代理不可用：{proxy}, 原因：{str(e)[:50]}")
        return False

# 测试示例
if __name__ == "__main__":
    # 从代理分享平台获取的 IP 列表
    proxy_list = [
        "http://112.14.47.6:52024",
        "http://180.122.151.186:3000",
        "http://223.241.78.143:8080"
    ]
    valid_proxies = []
    for proxy in proxy_list:
        if check_proxy(proxy):
            valid_proxies.append(proxy)
    print(f"可用代理列表：{valid_proxies}")

import scrapy

class ProxyTestSpider(scrapy.Spider):
    name = "proxy_test"
    allowed_domains = ["httpbin.org"]
    start_urls = ["http://httpbin.org/ip"]

    def start_requests(self):
        # 单个代理示例
        proxy = "http://112.14.47.6:52024"
        for url in self.start_urls:
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                meta={"proxy": proxy} # 核心配置：添加 proxy 元数据
            )

    def parse(self, response):
        # 打印返回结果，验证出口 IP 是否为代理 IP
        self.logger.info(f"响应结果：{response.text}")

import random
from scrapy import signals

class ProxyMiddleware:
    """自定义代理中间件，实现随机切换代理 IP"""
    def __init__(self, proxy_list):
        # 初始化代理池
        self.proxy_list = proxy_list

    @classmethod
    def from_crawler(cls, crawler):
        # 从配置文件读取代理列表
        proxy_list = crawler.settings.getlist("PROXY_LIST")
        return cls(proxy_list)

    def process_request(self, request, spider):
        """在请求发送前添加代理"""
        if self.proxy_list:
            # 随机选择一个代理
            proxy = random.choice(self.proxy_list)
            request.meta["proxy"] = proxy
            spider.logger.info(f"当前使用代理：{proxy}")
        return None

    def process_response(self, request, response, spider):
        """处理响应，若返回异常状态码则更换代理重试"""
        # 常见的封禁状态码：403/407/503/504
        if response.status in [403, 407, 503, 504]:
            spider.logger.warning(f"代理失效，状态码：{response.status}")
            # 更换代理重新请求
            if self.proxy_list:
                new_proxy = random.choice(self.proxy_list)
                request.meta["proxy"] = new_proxy
                return request.copy() # 重新发起请求
        return response

    def process_exception(self, request, exception, spider):
        """处理请求异常，更换代理重试"""
        spider.logger.error(f"请求异常：{exception}")
        if self.proxy_list:
            new_proxy = random.choice(self.proxy_list)
            request.meta["proxy"] = new_proxy
            return request.copy()

# 1. 启用自定义代理中间件
DOWNLOADER_MIDDLEWARES = {
    # 禁用 Scrapy 默认的代理中间件（如果有的话）
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': None,
    # 启用自定义代理中间件（优先级建议设置为 750，在 UserAgent 中间件之后）
    'your_project_name.middlewares.ProxyMiddleware': 750,
}

# 2. 配置可用的代理列表（替换为你筛选后的代理）
PROXY_LIST = [
    "http://112.14.47.6:52024",
    "http://180.122.151.186:3000",
    "http://223.241.78.143:8080",
]

# 3. 其他防封配置（重要）
# 禁用 Cookie，避免被识别为同一用户
COOKIES_ENABLED = False
# 设置下载延迟，避免请求过快
DOWNLOAD_DELAY = 2
# 随机化下载延迟（1-3 秒）
RANDOMIZE_DOWNLOAD_DELAY = True
# 设置请求头，模拟浏览器
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
# 重试次数
RETRY_TIMES = 3
# 重试的状态码
RETRY_HTTP_CODES = [403, 407, 500, 502, 503, 504]

import scrapy

class ProxyPoolSpider(scrapy.Spider):
    name = "proxy_pool"
    allowed_domains = ["httpbin.org"]
    start_urls = ["http://httpbin.org/ip"]

    def parse(self, response):
        self.logger.info(f"出口 IP: {response.text}")
        # 可以继续请求其他链接，验证代理自动切换
        yield scrapy.Request(
            url="http://httpbin.org/get",
            callback=self.parse_get
        )

    def parse_get(self, response):
        self.logger.info(f"GET 请求响应：{response.text[:200]}")

import random
import requests
from scrapy import signals
from lxml import etree

class DynamicProxyMiddleware:
    """动态获取免费代理的中间件"""
    def __init__(self):
        self.proxy_list = [] # 初始化时抓取一次代理
        self.refresh_proxy_list()

    def refresh_proxy_list(self):
        """从代理分享平台抓取并筛选可用代理"""
        # 从代理分享平台抓取高匿代理，如果提供免费接口的可以直接获取更方便
        url = "https://www.66daili.com/get-ip/"
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        }
        try:
            response = requests.get(url, headers=headers, timeout=10)
            html = etree.HTML(response.text)
            # 解析代理 IP 和端口
            ip_list = html.xpath('//tr/td[2]/text()')
            port_list = html.xpath('//tr/td[3]/text()')
            # 拼接代理并筛选
            temp_proxies = [f"http://{ip}:{port}" for ip, port in zip(ip_list, port_list)]
            self.proxy_list = [p for p in temp_proxies if self.check_proxy(p)]
            print(f"更新代理池，可用代理数量：{len(self.proxy_list)}")
        except Exception as e:
            print(f"获取代理失败：{e}")

    def check_proxy(self, proxy):
        """检测代理有效性"""
        test_url = "http://httpbin.org/ip"
        try:
            resp = requests.get(test_url, proxies={"http": proxy}, timeout=3)
            return resp.status_code == 200
        except:
            return False

    def process_request(self, request, spider):
        """每次请求前检查代理池，为空则刷新"""
        if not self.proxy_list:
            self.refresh_proxy_list()
        if self.proxy_list:
            proxy = random.choice(self.proxy_list)
            request.meta["proxy"] = proxy
        return None

2026-02-25 10:00:00 [proxy_pool] INFO: 当前使用代理：http://112.14.47.6:52024
2026-02-25 10:00:02 [proxy_pool] INFO: 出口 IP: {"origin":"112.14.47.6"}
2026-02-25 10:00:04 [proxy_pool] INFO: 当前使用代理：http://180.122.151.186:3000
2026-02-25 10:00:06 [proxy_pool] INFO: GET 请求响应：{"args":{},"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Language":"zh-CN,zh;q=0.9,en;q=0.8","Host":"httpbin.org","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36","X-Amzn-Trace-Id":"Root=1-67e3b8c6-1234567890abcdef12345678"},"origin":"180.122.151.186","url":"http://httpbin.org/get"}