多模态模型开发实战：文本、图像与语音融合指南

from transformers import CLIPImageProcessor
from PIL import Image

image_processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-base-patch32")

def preprocess_image(image_paths, target_size=(224, 224)):
    images = [Image.open(path).convert("RGB") for path in image_paths]
    inputs = image_processor(images, resize_size=target_size, crop_size=target_size,
                             normalize={"mean": [0.48145466, 0.4578275, 0.40821073],
                                        "std": [0.26862954, 0.26130258, 0.27577711]},
                             return_tensors="pt")
    return inputs["pixel_values"]

文本 - 语音数据预处理

语音数据需转换为梅尔频谱图。使用 librosa 提取特征时，注意统一采样率（如 16kHz）并进行降噪处理。

import librosa
import torch
import numpy as np

def preprocess_audio(audio_paths, sample_rate=16000, n_mels=80, max_length=3000):
    features = []
    for path in audio_paths:
        y, sr = librosa.load(path, sr=sample_rate)
        y, _ = librosa.effects.trim(y, top_db=20)
        mel_spec = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=n_mels, fmax=8000)
        log_mel_spec = librosa.power_to_db(mel_spec, ref=np.max)
        seq_len = log_mel_spec.shape[1]
        if seq_len > max_length:
            log_mel_spec = log_mel_spec[:, :max_length]
        else:
            pad_len = max_length - seq_len
            log_mel_spec = np.pad(log_mel_spec, ((0, 0), (0, pad_len)), mode="constant")
        features.append(torch.tensor(log_mel_spec, dtype=torch.float32))
    return torch.stack(features)

多模态模型开发实战：三大典型场景落地

场景一：跨模态问答系统

核心需求是结合图像与问题生成回答。我们选用 LLaVA-7B 模型，它结合了 CLIP 图像编码器和 LLaMA 语言模型。

from transformers import LlavaProcessor, LlavaForConditionalGeneration
import torch

model_name = "liuhaotian/LLaVA-7B-v1.5"
processor = LlavaProcessor.from_pretrained(model_name)
model = LlavaForConditionalGeneration.from_pretrained(
    model_name, torch_dtype=torch.float16, load_in_8bit=True, device_map="auto", trust_remote_code=True
)

推理时需将图像和问题一同输入，并控制生成参数以保证准确性。

from PIL import Image

def multimodal_qa(image_path, question, max_new_tokens=200, temperature=0.3):
    image = Image.open(image_path).convert("RGB")
    inputs = processor(text=question, images=image, return_tensors="pt", padding=True, truncation=True).to(model.device)
    with torch.no_grad():
        outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, temperature=temperature, top_p=0.9, do_sample=True, pad_token_id=processor.tokenizer.eos_token_id)
    answer = processor.decode(outputs[0], skip_special_tokens=True)
    return answer.split("ASSISTANT:")[-1].strip()

场景二：文生图生成系统

使用 Stable Diffusion 生成高质量图像。关键在于提示词（Prompt）工程和显存优化。

from transformers import StableDiffusionPipeline
import torch

model_name = "runwayml/stable-diffusion-v1-5"
pipe = StableDiffusionPipeline.from_pretrained(
    model_name, torch_dtype=torch.float16, use_safetensors=True, device_map="auto"
)
pipe.safety_checker = None
pipe.enable_attention_slicing()
pipe.enable_xformers_memory_efficient_attention()

通过 Gradio 可以快速构建交互界面，支持风格控制和参数调优。

场景三：多模态语音助手

整合 ASR（语音转文字）、LLM（文本理解）和 TTS（文字转语音）模块。

from transformers import WhisperProcessor, WhisperForConditionalGeneration
import librosa

asr_model_name = "openai/whisper-small"
asr_processor = WhisperProcessor.from_pretrained(asr_model_name)
asr_model = WhisperForConditionalGeneration.from_pretrained(asr_model_name, device_map="auto", torch_dtype=torch.float16)
asr_model.config.forced_decoder_ids = asr_processor.get_decoder_prompt_ids(language="zh", task="transcribe")

def speech_to_text(audio_path, sample_rate=16000):
    audio, sr = librosa.load(audio_path, sr=sample_rate)
    inputs = asr_processor(audio, sampling_rate=sr, return_tensors="pt", padding=True).to(asr_model.device)
    with torch.no_grad():
        outputs = asr_model.generate(**inputs, max_new_tokens=200)
    return asr_processor.decode(outputs[0], skip_special_tokens=True)

多模态模型训练微调与优化

通用模型往往难以满足行业专属需求，微调是关键。以医疗影像问答为例，采用 QLoRA 技术可在消费级 GPU 上完成微调。

微调数据准备

数据集需包含'图像 - 问题 - 回答'三元组，格式统一且语义对齐。

from datasets import Dataset
import json
import os

def load_medical_qa_dataset(data_path):
    samples = []
    with open(data_path, "r", encoding="utf-8") as f:
        for line in f:
            sample = json.loads(line)
            if not os.path.exists(sample["image_path"]):
                continue
            samples.append(sample)
    dataset = Dataset.from_list(samples)
    return dataset.train_test_split(test_size=0.1, seed=42)

QLoRA 微调实现

利用 PEFT 库配置 LoRA 参数，仅训练少量适配器，大幅降低显存占用。

from peft import LoraConfig, get_peft_model
from transformers import TrainingArguments
from trl import SFTTrainer

lora_config = LoraConfig(r=8, lora_alpha=32, target_modules=["q_proj", "v_proj", "k_proj", "o_proj"], lora_dropout=0.05, bias="none", task_type="CAUSAL_LM")
model = get_peft_model(model, lora_config)

training_args = TrainingArguments(output_dir="./llava-medical-qa-lora", per_device_train_batch_size=4, gradient_accumulation_steps=4, learning_rate=2e-4, num_train_epochs=5, fp16=True)
trainer = SFTTrainer(model=model, args=training_args, train_dataset=processed_dataset["train"], eval_dataset=processed_dataset["validation"], peft_config=lora_config, tokenizer=processor.tokenizer, max_seq_length=512)
trainer.train()

总结与建议

多模态开发的核心在于模态对齐与特征融合。理解类任务选 CLIP，生成类选 Stable Diffusion，复杂对话选混合架构。低资源场景下 QLoRA 是首选。部署时需兼顾性能与体验，注意合规风险。实际项目中应优先通过开源模型验证需求，再通过微调适配私有数据。