Python 构建 NLP 模型实战：从预处理到部署

文本预处理

原始文本包含 HTML 标签、标点符号和停用词，直接输入模型效果不佳。我们需要一个清理函数。

import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

# 下载必要的 NLTK 资源
nltk.download('stopwords')
nltk.download('punkt')

def clean_text(text):
    # 移除 HTML 标签
    text = re.sub(r'<.*?>', '', text)
    # 只保留字母并转小写
    text = re.sub(r'[^a-zA-Z]', ' ', text).lower()
    # 分词
    tokens = word_tokenize(text)
    # 去除停用词
    tokens = [word for word in tokens if word not in stopwords.words('english')]
    return ' '.join(tokens)

# 应用处理
train_texts = train_texts.apply(clean_text)
test_texts = test_texts.apply(clean_text)

特征工程

在尝试深度学习之前，可以先用传统方法提取特征作为基线。

TF-IDF

适用于逻辑回归等线性模型。

from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer(max_features=5000)
X_train = vectorizer.fit_transform(train_texts)
X_test = vectorizer.transform(test_texts)

Word2Vec

使用 gensim 训练词向量，捕捉语义信息。

from gensim.models import Word2Vec

# 将文本按空格分割成句子列表
sentences = [text.split() for text in train_texts]
word2vec_model = Word2Vec(sentences, vector_size=100, window=5, min_count=5, workers=4)
word2vec_model.save("word2vec.model")

搭建 NLP 模型

逻辑回归

基于 TF-IDF 特征的简单基线模型。

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

model = LogisticRegression()
model.fit(X_train, train_labels)

preds = model.predict(X_test)
print(f"Logistic Regression Accuracy: {accuracy_score(test_labels, preds)}")

LSTM 深度学习模型

LSTM 适合处理序列数据，能捕捉上下文依赖。注意这里需要构建自定义的数据加载器。

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset

# 定义 LSTM 模型结构
class LSTMModel(nn.Module):
    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim):
        super(LSTMModel, self).__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        embedded = self.embedding(x)
        _, (hidden, _) = self.lstm(embedded)
        return self.fc(hidden.squeeze(0))

# 超参数设置
VOCAB_SIZE = 5000
EMBEDDING_DIM = 100
HIDDEN_DIM = 128
OUTPUT_DIM = 1

model = LSTMModel(VOCAB_SIZE, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)

# 模拟数据加载（实际项目中需将文本转为索引）
# 此处演示训练循环结构
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 假设已有编码后的数据 tensor_data 和 labels_tensor
# 实际使用时请替换为真实的 Tokenized 数据
for epoch in range(5):
    model.train()
    optimizer.zero_grad()
    # 注意：实际训练中应传入真实编码后的批次数据
    # outputs = model(batch_input) 
    # loss = criterion(outputs.squeeze(), batch_labels.float())
    # loss.backward()
    # optimizer.step()
    print(f"Epoch {epoch+1}, Training Loop Ready")

使用预训练的 BERT 模型

BERT 在大多数 NLP 任务上表现优异，无需从头训练词向量。

from transformers import BertTokenizer, BertForSequenceClassification
from torch.utils.data import DataLoader

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased')

# Tokenize 数据
train_encodings = tokenizer(list(train_texts), truncation=True, padding=True, max_length=512, return_tensors="pt")
test_encodings = tokenizer(list(test_texts), truncation=True, padding=True, max_length=512, return_tensors="pt")

# 转换为 PyTorch Dataset
class IMDbDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __len__():
        return len(self.labels)

    def __getitem__(self, idx):
        item = {key: val[idx] for key, val in self.encodings.items()}
        item["labels"] = torch.tensor(self.labels[idx])
        return item

train_dataset = IMDbDataset(train_encodings, list(train_labels))
test_dataset = IMDbDataset(test_encodings, list(test_labels))

# 训练 BERT
train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)
optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5)

model.train()
for epoch in range(3):
    for batch in train_loader:
        optimizer.zero_grad()
        outputs = model(**batch)
        loss = outputs.loss
        loss.backward()
        optimizer.step()
    print(f"Epoch {epoch+1}, Loss: {loss.item()}")

模型评估

使用分类报告查看详细性能指标。

from sklearn.metrics import classification_report

model.eval()
preds = []
with torch.no_grad():
    for batch in test_dataset:
        output = model(**batch)
        preds.append(torch.argmax(output.logits, axis=1).numpy())

print(classification_report(test_labels, preds))

部署模型

使用 FastAPI 将模型封装为 HTTP 服务。

from fastapi import FastAPI
import torch

app = FastAPI()

@app.post("/predict/")
def predict(text: str):
    encoding = tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=512)
    with torch.no_grad():
        output = model(**encoding)
        pred = torch.argmax(output.logits, axis=1).item()
    return {"sentiment": "positive" if pred == 1 else "negative"}

运行服务：

uvicorn main:app --reload

总结

本文梳理了从环境配置到模型部署的完整链路。在实际业务中，建议优先尝试 BERT 等预训练模型以获得更好的效果，若资源受限可考虑 TF-IDF + 逻辑回归的轻量方案。根据具体场景选择合适的模型架构，平衡精度与推理成本是关键。