Ubuntu 下 AMD AI MAX 395 使用 ROCm 部署 Qwen 模型

二、Docker 环境准备（vLLM）

1. 安装并配置 Docker

为了加速镜像拉取，我们配置阿里云源及镜像加速器。

# 更新系统源
sudo apt update -y

# 安装 Docker 依赖
sudo apt-get install apt-transport-https ca-certificates curl software-properties-common lrzsz -y

# 添加 Docker GPG 密钥
sudo curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -

# 添加 Docker 软件源
sudo add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
sudo apt update -y

# 安装 Docker CE
sudo apt-get install docker-ce -y

# 配置镜像加速器
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": [
    "https://docker.1panel.live",
    "https://hub.rat.dev"
  ]
}
EOF

# 重载配置并重启服务
sudo systemctl daemon-reload
sudo systemctl restart docker

2. 拉取 vLLM 镜像

如果主机无法联网，可先在有网机器上打包镜像到 U 盘，再导入本地。

2.1 打包镜像

# 在有网的主机上拉取镜像（版本可选）
docker pull rocm/vllm:rocm7.0.0_vllm_0.11.2

# 导出为 tar 文件
docker save -o vllm_rocm7.tar rocm/vllm:rocm7.0.0_vllm_0.11.2

# 将 vllm_rocm7.tar 复制到 U 盘

2.2 加载镜像

# 假设 U 盘挂载在 /media/user/MyUSB
# 拷贝到本地目录
cp /media/user/MyUSB/vllm_rocm7.tar ~/

# 导入镜像
docker load -i vllm_rocm7.tar

# 验证
docker images

三、千问模型部署

1. Qwen3-32B 对话模型

1.1 下载模型

# 创建目录
mkdir -p /home/user/models/Qwen-32B-AWQ
export MODEL_DIR=/home/user/models/Qwen-32B-AWQ

# 安装 modelscope
pip3 install modelscope

# 下载模型
python3 -c """
import os
from modelscope.hub.snapshot_download import snapshot_download
model_id = 'qwen/Qwen-32B-AWQ'
snapshot_download(
    model_id=model_id,
    cache_dir=os.environ.get('MODEL_DIR'),
    revision='master'
)
"""

1.2 启动模型

注意：AMD Ryzen AI Max 395+ 的 GPU 代号为 gfx1151，但部分 ROCm 版本可能未完全支持，需通过环境变量强制指定。

docker run -it \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-32B-AWQ/qwen/Qwen3-32B-AWQ:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 \
vllm serve /model \
--quantization awq \
--dtype float16 \
--served-model-name Qwen3-32B-AWQ \
--trust-remote-code \
--max-model-len 8192

1.3 验证模型

curl http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{ "model": "Qwen3-32B-AWQ", "prompt": "你是谁？", "max_tokens": 2000, "temperature": 0.7 }'

2. Qwen3-Embedding 向量模型

2.1 下载模型

python3 -c """
from modelscope import snapshot_download
snapshot_download('Qwen/Qwen3-Embedding-8B', cache_dir='/home/user/models')
"""

2.2 启动模型

docker run -d \
--name vllm-embedding \
--restart=always \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-Embedding-8B:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 \
vllm serve /model \
--port 8001 \
--task embed \
--dtype float16 \
--max-model-len 8192 \
--gpu-memory-utilization 0.4 \
--trust-remote-code \
--served-model-name qwen-embedding

2.3 验证模型

# 查看容器状态
docker ps

# 查看日志
docker logs vllm-embedding

# 发送测试请求
curl http://localhost:8001/v1/embeddings \
-H "Content-Type: application/json" \
-d '{ "model": "qwen-embedding", "input": "你好，测试一下向量化服务" }'

3. Qwen3-Reranker 重排序模型

由于 vLLM 目前尚未原生支持 Rerank 任务，我们需要构建一个自定义服务。

3.1 下载模型

python3 -c """
from modelscope import snapshot_download
snapshot_download('Qwen/Qwen3-Reranker-8B', cache_dir='/home/user/models')
"""

3.2 配置启动脚本与 uv 管理

创建项目目录并编写 FastAPI 服务脚本。

mkdir -p /home/user/qwen_project
cd /home/user/qwen_project
nano rerank_service.py

rerank_service.py

import torch
import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from typing import List, Optional
from transformers import AutoModelForCausalLM, AutoTokenizer

MODEL_PATH = "/model"
PORT = 8002

app = FastAPI()
print(f"Loading model from {MODEL_PATH} ...")

tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, padding_side='left', trust_remote_code=True)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_PATH, trust_remote_code=True, device_map="auto", torch_dtype=torch.float16, attn_implementation="flash_attention_2"
).eval()

token_false_id = tokenizer.convert_tokens_to_ids("no")
token_true_id = tokenizer.convert_tokens_to_ids("yes")

prefix = "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n"
suffix = "<|im_end|>\n<|im_start|>assistant\n\n\n"

prefix_tokens = tokenizer.encode(prefix, add_special_tokens=False)
suffix_tokens = tokenizer.encode(suffix, add_special_tokens=False)
max_length = 8192

print("Model loaded successfully!")

def format_instruction(instruction, query, doc):
    if instruction is None or instruction == "":
        instruction = 'Given a web search query, retrieve relevant passages that answer the query'
    return "<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {doc}".format(instruction=instruction, query=query, doc=doc)

def process_inputs(pairs):
    inputs = tokenizer(pairs, padding=False, truncation='longest_first', return_attention_mask=False, max_length=max_length - len(prefix_tokens) - len(suffix_tokens))
    for i, ele in enumerate(inputs['input_ids']):
        inputs['input_ids'][i] = prefix_tokens + ele + suffix_tokens
    inputs = tokenizer.pad(inputs, padding=True, return_tensors="pt", max_length=max_length)
    for key in inputs:
        inputs[key] = inputs[key].to(model.device)
    return inputs

@torch.no_grad()
def compute_scores(inputs):
    batch_scores = model(**inputs).logits[:, -1, :]
    true_vector = batch_scores[:, token_true_id]
    false_vector = batch_scores[:, token_false_id]
    batch_scores = torch.stack([false_vector, true_vector], dim=1)
    batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
    scores = batch_scores[:, 1].exp().tolist()
    return scores

class RerankRequest(BaseModel):
    model: str = "qwen-reranker"
    query: str
    documents: List[str]
    top_n: Optional[int] = None
    instruction: Optional[str] = None

@app.post("/v1/rerank")
async def rerank(request: RerankRequest):
    try:
        query = request.query
        documents = request.documents
        instruction = request.instruction
        if not documents:
            return {"results": []}
        pairs = [format_instruction(instruction, query, doc) for doc in documents]
        inputs = process_inputs(pairs)
        scores = compute_scores(inputs)
        results = []
        for i, score in enumerate(scores):
            results.append({"index": i, "relevance_score": float(score), "document": documents[i]})
        results.sort(key=lambda x: x["relevance_score"], reverse=True)
        if request.top_n:
            results = results[:request.top_n]
        return {"model": request.model, "results": results, "usage": {"total_tokens": inputs.input_ids.numel()}}
    except Exception as e:
        print(f"Error: {e}")
        raise HTTPException(status_code=500, detail=str(e))

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=PORT)

pyproject.toml

[project]
name = "qwen3-reranker-service"
version = "0.1.0"
description = "Rerank service using Qwen3 and ROCm"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
    "transformers>=4.51.0",
    "fastapi",
    "uvicorn",
    "modelscope",
    "accelerate",
    "pydantic"
]

注：此处不写 torch 依赖，直接使用系统自带的 AMD 版本。

3.3 构建镜像

将代码与依赖打包进镜像，方便迁移。

# 启动临时构建容器
docker run -it --name builder --network=host -v /home/user/qwen_project:/tmp_build rocm/vllm:rocm7.0.0_vllm_0.11.2_20251210 bash

# 在容器内操作
pip install uv -i https://pypi.tuna.tsinghua.edu.cn/simple
mkdir -p /app
cp /tmp_build/rerank_service.py /app/
cp /tmp_build/pyproject.toml /app/
cd /app

# 使用 uv 安装依赖到系统环境
uv pip install --system -r pyproject.toml -i https://pypi.tuna.tsinghua.edu.cn/simple

# 验证
pip list | grep transformers
exit

# 提交为新镜像
docker commit builder qwen-rerank:v1
docker rm builder

# 导出镜像（可选）
docker save -o qwen-rerank-v1.tar qwen-rerank:v1

3.4 启动与检验

# 启动服务
docker run -d \
--name final_reranker \
--restart=always \
--network=host \
--group-add=video \
--ipc=host \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--device /dev/kfd \
--device /dev/dri \
-v /home/user/models/Qwen3-Reranker-8B:/model \
-e HSA_OVERRIDE_GFX_VERSION=11.0.0 \
qwen-rerank:v1 \
python3 /app/rerank_service.py

# 验证
curl http://localhost:8002/v1/rerank \
-H "Content-Type: application/json" \
-d '{ "model": "qwen-reranker", "query": "中国的首都在哪里？", "documents": [ "重力是万有引力。", "中国的首都是北京。", "香蕉很好吃。" ] }'