Ubuntu 下 AMD AI MAX 395+ 使用 ROCm 加速部署 Qwen 模型

2. 拉取 vLLM 镜像

为满足离线部署需求，可将镜像打包至 U 盘。若无此需求可直接在目标电脑上拉取。

2.1 将镜像文件打包进 U 盘

# 1. 拉取镜像（版本可选）
docker pull rocm/vllm:rocm7.0.0_vllm_0.11.2

# 2. 保存为 tar 文件
docker save -o vllm_rocm7.tar rocm/vllm:rocm7.0.0_vllm_0.11.2

# 3. 复制文件到 U 盘（确保格式是 exFAT 或 NTFS）

2.2 加载镜像

# 1. 找到 U 盘路径（通常在 /media/用户名/U 盘名称 下）
# 2. 拷贝镜像文件到本地
cp /media/用户名/U 盘名称/vllm_rocm7.tar ~/

# 3. 导入镜像
docker load -i vllm_rocm7.tar

# 4. 验证镜像存在
docker images

三、千问模型部署

1. Qwen3-32B

1.1 下载模型

# 1. 按需创建目录
mkdir -p /home/user/models/Qwen-32B-AWQ
export MODEL_DIR=/home/user/models/Qwen-32B-AWQ

# 2. 安装/升级 modelscope 依赖
pip3 install modelscope

# 3. 下载 Qwen-32B-AWQ 模型
python3 -c """
import os
from modelscope.hub.snapshot_download import snapshot_download

model_id = 'qwen/Qwen-32B-AWQ'
snapshot_download(
    model_id=model_id,
    cache_dir=os.environ.get('MODEL_DIR'),
    revision='master'
)
"""

1.2 启动模型

docker run -it \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-32B-AWQ/qwen/Qwen3-32B-AWQ:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 \
vllm serve /model \
--quantization awq \
--dtype float16 \
--served-model-name Qwen3-32B-AWQ \
--trust-remote-code \
--max-model-len 8192

1.3 验证模型

curl http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{ "model": "Qwen3-32B-AWQ", "prompt": "你是谁？", "max_tokens": 2000, "temperature": 0.7 }'

2. Qwen3-Embedding

2.1 下载模型

python3 -c """
from modelscope import snapshot_download
snapshot_download('Qwen/Qwen3-Embedding-8B', cache_dir='/home/user/models')
"""

2.2 启动模型

docker run -d \
--name vllm-embedding \
--restart=always \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-Embedding-8B:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 \
vllm serve /model \
--port 8001 \
--task embed \
--dtype float16 \
--max-model-len 8192 \
--gpu-memory-utilization 0.4 \
--trust-remote-code \
--served-model-name qwen-embedding

2.3 验证模型

# 查看容器状态
docker ps

# 查看日志
docker logs vllm-embedding

# 发送测试请求
curl http://localhost:8001/v1/embeddings \
-H "Content-Type: application/json" \
-d '{ "model": "qwen-embedding", "input": "你好，测试一下向量化服务" }'

3. Qwen3-Reranker

3.1 下载模型

python3 -c """
from modelscope import snapshot_download
snapshot_download('Qwen/Qwen3-Reranker-8B', cache_dir='/home/user/models')
"""

3.2 配置启动脚本与 uv 管理

因 vLLM 暂不支持直接运行 Qwen3 Rerank 模型，需手动编写脚本。参考魔塔社区示例：https://www.modelscope.cn/models/Qwen/Qwen3-Reranker-8B

# 创建项目目录
mkdir -p /home/user/qwen_project
cd /home/user/qwen_project

# 创建 Python 脚本
nano rerank_service.py

Python 脚本内容：

import torch
import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import List, Optional
from transformers import AutoModelForCausalLM, AutoTokenizer

# === 配置区域 ===
MODEL_PATH = "/model"
PORT = 8002

app = FastAPI()
print(f"Loading model from {MODEL_PATH} ...")

# 1. 加载 Tokenizer
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, padding_side='left', trust_remote_code=True)

# 2. 加载模型
model = AutoModelForCausalLM.from_pretrained(
    MODEL_PATH,
    trust_remote_code=True,
    device_map="auto",
    torch_dtype=torch.float16,
    attn_implementation="flash_attention_2"
).eval()

# 3. 准备 Token IDs
token_false_id = tokenizer.convert_tokens_to_ids("no")
token_true_id = tokenizer.convert_tokens_to_ids("yes")

# 4. 准备前后缀
prefix = "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n"
suffix = "<|im_end|>\n<|im_start|>assistant\n\n\n"
prefix_tokens = tokenizer.encode(prefix, add_special_tokens=False)
suffix_tokens = tokenizer.encode(suffix, add_special_tokens=False)
max_length = 8192
print("Model loaded successfully!")

# === 核心处理逻辑 ===
def format_instruction(instruction, query, doc):
    if instruction is None or instruction == "":
        instruction = 'Given a web search query, retrieve relevant passages that answer the query'
    return "<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {doc}".format(instruction=instruction, query=query, doc=doc)

def process_inputs(pairs):
    inputs = tokenizer(
        pairs,
        padding=False,
        truncation='longest_first',
        return_attention_mask=False,
        max_length=max_length - len(prefix_tokens) - len(suffix_tokens)
    )
    for i, ele in enumerate(inputs['input_ids']):
        inputs['input_ids'][i] = prefix_tokens + ele + suffix_tokens
    inputs = tokenizer.pad(inputs, padding=True, return_tensors="pt", max_length=max_length)
    for key in inputs:
        inputs[key] = inputs[key].to(model.device)
    return inputs

@torch.no_grad()
def compute_scores(inputs):
    batch_scores = model(**inputs).logits[:, -1, :]
    true_vector = batch_scores[:, token_true_id]
    false_vector = batch_scores[:, token_false_id]
    batch_scores = torch.stack([false_vector, true_vector], dim=1)
    batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
    scores = batch_scores[:, 1].exp().tolist()
    return scores

# === API 定义 ===
class RerankRequest(BaseModel):
    model: str = "qwen-reranker"
    query: str
    documents: List[str]
    top_n: Optional[int] = None
    instruction: Optional[str] = None

@app.post("/v1/rerank")
async def rerank(request: RerankRequest):
    try:
        query = request.query
        documents = request.documents
        instruction = request.instruction
        if not documents:
            return {"results": []}
        pairs = [format_instruction(instruction, query, doc) for doc in documents]
        inputs = process_inputs(pairs)
        scores = compute_scores(inputs)
        results = []
        for i, score in enumerate(scores):
            results.append({
                "index": i,
                "relevance_score": float(score),
                "document": documents[i]
            })
        results.sort(key=lambda x: x["relevance_score"], reverse=True)
        if request.top_n:
            results = results[:request.top_n]
        return {
            "model": request.model,
            "results": results,
            "usage": {"total_tokens": inputs.input_ids.numel()}
        }
    except Exception as e:
        print(f"Error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=PORT)

创建 pyproject.toml 管理依赖：

[project]
name = "qwen3-reranker-service"
version = "0.1.0"
description = "Rerank service using Qwen3 and ROCm"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
    "transformers>=4.51.0",
    "fastapi",
    "uvicorn",
    "modelscope",
    "accelerate",
    "pydantic"
]

3.3 启动镜像

将依赖环境和业务代码打包进镜像。

# 启动临时构建容器
docker run -it --name builder \
--network=host \
-v /home/user/qwen_project:/tmp_build \
rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 \
bash

# 进入容器后操作
# 1. 安装 uv
pip install uv -i https://pypi.tuna.tsinghua.edu.cn/simple

# 2. 创建应用目录
mkdir -p /app

# 3. 复制代码到容器内部
cp /tmp_build/rerank_service.py /app/
cp /tmp_build/pyproject.toml /app/

# 4. 进入应用目录
cd /app

# 5. 使用 uv 安装依赖
uv pip install --system -r pyproject.toml -i https://pypi.tuna.tsinghua.edu.cn/simple

# 6. 验证
pip list | grep transformers
ls /app

# 7. 退出
exit

提交为新镜像：

docker commit builder qwen-rerank:v1
docker rm builder
docker save -o qwen-rerank-v1.tar qwen-rerank:v1

启动最终服务：

docker run -d \
--name final_reranker \
--restart=always \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-Reranker-8B:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
qwen-rerank:v1 \
python3 /app/rerank_service.py

3.4 检验模型

# 查看容器状态
docker ps

# 查看日志
docker logs final_reranker

# 访问 Swagger UI
curl http://localhost:8002/docs

# 终端发送请求
curl http://localhost:8002/v1/rerank \
-H "Content-Type: application/json" \
-d '{ "model": "qwen-reranker", "query": "中国的首都在哪里？", "documents": [ "重力是万有引力。", "中国的首都是北京。", "香蕉很好吃。" ] }'