Python 十大实用技巧：爬虫、自动化与数据处理

Python 十大实用技巧：爬虫、自动化与数据处理 | 极客日志

import urllib3

def download_content(url):
    """下载网页内容"""
    http = urllib3.PoolManager()
    try:
        response = http.request("GET", url)
        return response.data.decode('utf-8')
    except Exception as e:
        print(f"下载失败：{e}")
        return None

def save_to_file(filename, content):
    """保存内容到文件"""
    if not content:
        return
    with open(filename, "w", encoding="utf-8") as fo:
        fo.write(content)

url = "https://zkaoy.com/sions/exam"
result = download_content(url)
save_to_file("tips1.html", result)

from bs4 import BeautifulSoup

def create_doc_from_filename(filename):
    """读取 HTML 文件并返回 BeautifulSoup 对象"""
    with open(filename, "r", encoding='utf-8') as fo:
        html_content = fo.read()
    return BeautifulSoup(html_content, 'html.parser')

doc = create_doc_from_filename("tips1.html")
post_list = doc.find_all("div", class_="post-info")
for post in post_list:
    links = post.find_all("a")
    if len(links) > 1:
        link_tag = links[1]
        title = link_tag.text.strip()
        href = link_tag.get("href")
        print(f"标题：{title}, 链接：{href}")

import urllib3
import pandas as pd

def download_html(url):
    http = urllib3.PoolManager()
    response = http.request("GET", url)
    return response.data.decode('utf-8')

url = "http://fx.cmbchina.com/Hq/"
html_content = download_html(url)

# 读取所有表格
table_list = pd.read_html(html_content)

# 假设我们需要第二个表格（索引为 1）
if len(table_list) > 1:
    df = table_list[1]
    # 保存为 Excel
    df.to_excel("tips2.xlsx", index=False)
    print("表格已保存至 tips2.xlsx")
else:
    print("未找到合适的表格")

import os
from bs4 import BeautifulSoup
from urllib.request import urlretrieve

def download_images(url, output_dir):
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    
    html = download_content(url)
    if not html:
        return
    
    save_to_file("temp_page.html", html)
    doc = create_doc_from_filename("temp_page.html")
    images = doc.find_all("img")
    
    count = 0
    for img in images:
        src = img.get("src")
        if src and src.startswith("http"):
            filename = src.split("/")[-1] or f"image_{count}.jpg"
            try:
                urlretrieve(src, os.path.join(output_dir, filename))
                count += 1
            except Exception as e:
                print(f"下载失败 {filename}: {e}")
    print(f"共下载 {count} 张图片")

url = "https://www.duitang.com/search/?kw=ins%E9%A3%8E%E8%83%8C%E6%99%AF%E5%9B%BE&type=feed"
download_images(url, "tips_3")

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False   # 用来正常显示负号

# 读取数据
df = pd.read_excel("tips2.xlsx")
# 过滤表头行
df = df[df.index > 0]

sns.set()
figure = plt.figure(figsize=(10, 5))
ax = figure.add_subplot(1, 1, 1)

# 绘图
plt.xlabel("类别", fontsize=14)
plt.ylabel("卖出价", fontsize=14)
plt.title("外汇情况统计", fontsize=14)

# 确保数值列正确转换
df[3] = pd.to_numeric(df[3], errors='coerce')

plt.bar(x=df[0], height=df[3].values, color='#1f77b4')
plt.xticks(rotation=90)
plt.show()

import jieba
import wordcloud
import matplotlib.pyplot as plt

# 读取文本
text = ""
with open("news_title.txt", "r", encoding="utf-8") as fo:
    text = fo.read()

# 分词
split_list = jieba.lcut(text)
final_text = " ".join(split_list)

# 停用词
stopwords = ["的", "是", "了", "在", "我"]

# 生成词云
wc = wordcloud.WordCloud(
    font_path="/System/Library/Fonts/PingFang.ttc",  # Windows 需改为对应字体路径
    width=1000,
    height=700,
    background_color="white",
    max_words=100,
    stopwords=stopwords
)
wc.generate(final_text)

plt.imshow(wc)
plt.axis("off")
plt.show()

import os

def batch_rename_files(folder_path, prefix="图片_"):
    files = sorted(os.listdir(folder_path))
    idx = 0
    for item in files:
        if item.lower().endswith(('.jpg', '.png')):
            old_path = os.path.join(folder_path, item)
            new_name = f"{prefix}{idx}.jpg"
            new_path = os.path.join(folder_path, new_name)
            try:
                os.rename(old_path, new_path)
                idx += 1
            except Exception as e:
                print(f"重命名失败 {item}: {e}")

batch_rename_files("tips_3")

import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageFont
import os

def add_watermark(input_folder, output_folder):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    
    for file in os.listdir(input_folder):
        if file.lower().endswith(('.jpg', '.png')):
            img_path = os.path.join(input_folder, file)
            img = cv2.imread(img_path)
            if img is not None:
                img_pil = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
                draw = ImageDraw.Draw(img_pil)
                
                # 字体设置
                try:
                    font = ImageFont.truetype("/System/Library/Fonts/PingFang.ttc", 50)
                except:
                    font = ImageFont.load_default()
                
                draw.text((100, 100), "版权水印", fill=(168, 121, 103), font=font)
                
                img_out = cv2.cvtColor(np.asarray(img_pil), cv2.COLOR_RGB2BGR)
                out_path = os.path.join(output_folder, file)
                cv2.imwrite(out_path, img_out)

def adjust_saturation(input_folder, output_folder, factor=1.5):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    
    for file in os.listdir(input_folder):
        if file.lower().endswith(('.jpg', '.png')):
            img_path = os.path.join(input_folder, file)
            pic = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
            if pic is not None:
                hsv = cv2.cvtColor(pic, cv2.COLOR_BGR2HSV)
                h, s, v = cv2.split(hsv)
                s_new = np.uint8(s * factor)
                merged = cv2.merge([h, s_new, v])
                res = cv2.cvtColor(merged, cv2.COLOR_HSV2BGR)
                cv2.imwrite(os.path.join(output_folder, file), res)

def resize_images(input_folder, output_folder, scale=0.5):
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)
    
    for file in os.listdir(input_folder):
        if file.lower().endswith(('.jpg', '.png')):
            img_path = os.path.join(input_folder, file)
            pic = cv2.imread(img_path)
            if pic is not None:
                h, w = pic.shape[:2]
                new_w, new_h = int(w * scale), int(h * scale)
                resized = cv2.resize(pic, (new_w, new_h))
                cv2.imwrite(os.path.join(output_folder, file), resized)

虚拟环境管理：使用 venv 或 conda 隔离项目依赖，避免库版本冲突。

python -m venv myenv
source myenv/bin/activate  # Linux/Mac
myenv\Scripts\activate     # Windows
pip install -r requirements.txt

异常处理：在网络请求和文件操作中，务必使用 try-except 块捕获潜在错误，防止程序崩溃。
编码规范：统一使用 UTF-8 编码处理文本文件，特别是在涉及中文文件名或内容时。
资源清理：下载临时文件后及时清理，避免占用过多磁盘空间。
反爬策略：在进行网络爬虫开发时，请遵守目标网站的 robots.txt 协议，控制请求频率，避免对服务器造成压力。

Python 十大实用技巧：爬虫、自动化与数据处理

1. Python 简介

1.1 什么是 Python？

1.2 发展历史

1.3 主要特点

2. Python 典型应用场景

3. 实战案例详解

3.1 爬取文档与资讯

第一步：下载网页并保存

第二步：解析 HTML 提取数据

3.2 抓取表格与数据分析

3.3 批量下载图片

3.4 数据可视化

3.5 文本分析与词云

3.6 文件批量管理

3.7 图片批量处理

添加水印

4. 开发环境与最佳实践

5. 总结

更多推荐文章

相关免费在线工具

Python 十大实用技巧：爬虫、自动化与数据处理

1. Python 简介

1.1 什么是 Python？

1.2 发展历史

1.3 主要特点

2. Python 典型应用场景

3. 实战案例详解

3.1 爬取文档与资讯

第一步：下载网页并保存

第二步：解析 HTML 提取数据

3.2 抓取表格与数据分析

3.3 批量下载图片

3.4 数据可视化

3.5 文本分析与词云

3.6 文件批量管理

3.7 图片批量处理

添加水印

4. 开发环境与最佳实践

5. 总结

微信扫一扫，关注极客日志

更多推荐文章

相关免费在线工具