基于 Qwen3-VL 与 LLaMA-Factory 的 Grounding 任务 LoRA 微调实战

微调实战

准备 LLaMA-Factory 环境

克隆项目并创建虚拟环境。注意卸载默认 CPU 版本的 PyTorch，根据 CUDA 版本安装 GPU 版。

git clone https://github.com/hiyouga/LLaMA-Factory
conda create -n llama-factory python=3.12 -y
conda activate llama-factory
pip install -e ".[torch,metrics]" --no-build-isolation
pip uninstall torch torchvision -y
# 以 CUDA 11.8 为例
pip install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118

数据集格式转换

LLaMA-Factory 需要特定格式的数据集。对于 Grounding 任务，Qwen3-VL 使用 0-1000 的相对坐标，而 YOLO 格式通常使用 0-1 的归一化坐标，因此需要进行转换。

下面脚本实现了从 YOLO 格式到 Qwen3-VL-Grounding 格式的转换，重点在于坐标系的缩放与 JSON 结构的构建。

import os
import json
from tqdm import tqdm

# ================== 配置区域 ==================
IMAGE_DIR = "images"          # 图片所在目录
LABEL_DIR = "labels"          # YOLO 标签所在目录
OUTPUT_JSON = "qwen3_vl_grounding_mllm.json" # 输出文件名
CLASS_ID2NAME = {              # 类别 ID 映射名称（可选）
    0: "house",
    # 1: "car",
}
USE_ONLY_CLASS_IDS = None     # 仅保留特定类别（可选）
USER_PROMPT = (
    "<image>\n"
    "Locate all objects in this image and output the bbox coordinates "
    "in JSON format using relative coordinates in the range [0, 1000]."
)
# =============================================

def yolo_to_xyxy_relative(xc, yc, w, h):
    """YOLO 归一化坐标 -> 相对坐标系四点 [x_min, y_min, x_max, y_max]"""
    x_min = xc - w / 2
    y_min = yc - h / 2
    x_max = xc + w / 2
    y_max = yc + h / 2
    x_min = max(0.0, min(1.0, x_min))
    y_min = max(0.0, min(1.0, y_min))
    x_max = max(0.0, min(1.0, x_max))
    y_max = max(0.0, min(1.0, y_max))
    return [x_min, y_min, x_max, y_max]

def scale_to_qwen_coords(xyxy_rel, scale=1000):
    """[0,1] -> [0, scale]，转换为 Qwen3-VL 整数坐标"""
    x_min, y_min, x_max, y_max = xyxy_rel
    return [
        int(round(x_min * scale)),
        int(round(y_min * scale)),
        int(round(x_max * scale)),
        int(round(y_max * scale)),
    ]

def collect_image_files(image_dir):
    exts = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
    files = []
    for fname in os.listdir(image_dir):
        if os.path.splitext(fname)[1].lower() in exts:
            files.append(fname)
    return sorted(files)

def main():
    image_files = collect_image_files(IMAGE_DIR)
    if not image_files:
        print(f"No images found in {IMAGE_DIR}")
        return

    dataset = []
    for img_name in tqdm(image_files, desc="Converting"):
        img_path = os.path.join(IMAGE_DIR, img_name)
        img_rel_or_abs = os.path.abspath(img_path)
        base, _ = os.path.splitext(img_name)
        label_path = os.path.join(LABEL_DIR, base + ".txt")

        if not os.path.exists(label_path):
            continue

        bboxes_qwen = []
        cls_ids = []

        with open(label_path, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                parts = line.split()
                if len(parts) < 5:
                    continue
                cls_id = int(parts[0])
                if USE_ONLY_CLASS_IDS is not None and cls_id not in USE_ONLY_CLASS_IDS:
                    continue
                xc, yc, w, h = float(parts[1]), float(parts[2]), float(parts[3]), float(parts[4])
                xyxy_rel = yolo_to_xyxy_relative(xc, yc, w, h)
                xyxy_qwen = scale_to_qwen_coords(xyxy_rel, scale=1000)
                bboxes_qwen.append(xyxy_qwen)
                cls_ids.append(cls_id)

        if not bboxes_qwen:
            continue

        objects = []
        for cid, box in zip(cls_ids, bboxes_qwen):
            obj = {"cls_id": cid, "bbox_2d": box}
            if cid in CLASS_ID2NAME:
                obj["cls_name"] = CLASS_ID2NAME[cid]
            objects.append(obj)

        answer_obj = {"objects": objects}
        answer_str = json.dumps(answer_obj, ensure_ascii=False)

        sample = {
            "conversations": [
                {"from": "human", "value": USER_PROMPT},
                {"from": "gpt", "value": answer_str}
            ],
            "images": [img_rel_or_abs]
        }
        dataset.append(sample)

    with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
        json.dump(dataset, f, ensure_ascii=False, indent=2)
    print(f"Done. Wrote {len(dataset)} samples to {OUTPUT_JSON}")

if __name__ == "__main__":
    main()

生成的 JSON 文件需放入 LLaMA-Factory/data 目录下。随后修改 dataset_info.json，添加自定义数据集配置：

"qwen3_vl_grounding_mllm": {
    "file_name": "qwen3_vl_grounding_mllm.json",
    "formatting": "sharegpt",
    "columns": {
        "messages": "conversations",
        "images": "images"
    }
}

启动训练

进入项目目录，启动可视化界面：

cd LLaMA-Factory
llamafactory-cli webui

在 WebUI 中配置训练参数。建议计算类型选择 Pure_bf16 以节省显存。具体参数包括语言设置、模型路径、下载源及计算精度等。完成配置后保存参数并点击开始训练。

训练完成后，可在对话测试模块加载微调后的模型，上传图片和提示词验证 Grounding 效果，确认无误后可导出模型权重。