Double DQN 算法详解：原理、流程与 PyTorch 实现

相比之下，传统 DQN 的目标值是：

$$y_t^{DQN} = r_t + \gamma \max_a Q_{\theta^-}(s_{t+1}, a)$$

这里的 max 操作是导致过估计问题的根源。

Python 实现

下面给出基于 PyTorch 框架的 Double DQN 完整实现，包含核心的在线网络和目标网络更新机制。

导入必要库

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import random
from collections import deque

超参数设置

# Hyperparameters
GAMMA = 0.99          # 折扣因子
LR = 0.001            # 学习率
BATCH_SIZE = 64       # 批量大小
MEMORY_CAPACITY = 10000  # 经验回放池容量
TARGET_UPDATE = 10    # 目标网络更新周期

定义网络结构

class QNetwork(nn.Module):
    def __init__(self, state_dim, action_dim):
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(state_dim, 128)
        self.fc2 = nn.Linear(128, 128)
        self.fc3 = nn.Linear(128, action_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

这里定义了三层全连接网络，中间层使用 ReLU 激活函数，输出层直接输出 Q 值。

经验回放池

class ReplayBuffer:
    def __init__(self, capacity):
        self.buffer = deque(maxlen=capacity)

    def push(self, state, action, reward, next_state, done):
        self.buffer.append((state, action, reward, next_state, done))

    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        states, actions, rewards, next_states, dones = zip(*batch)
        return (np.array(states), np.array(actions), np.array(rewards), 
                np.array(next_states), np.array(dones))

    def __len__(self):
        return len(self.buffer)

Double DQN 智能体

class DoubleDQNAgent:
    def __init__(self, state_dim, action_dim):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.online_net = QNetwork(state_dim, action_dim)
        self.target_net = QNetwork(state_dim, action_dim)
        self.target_net.load_state_dict(self.online_net.state_dict())
        self.target_net.eval()
        self.optimizer = optim.Adam(self.online_net.parameters(), lr=LR)
        self.memory = ReplayBuffer(MEMORY_CAPACITY)
        self.steps_done = 0

    def select_action(self, state, epsilon):
        if random.random() < epsilon:
            return random.randint(0, self.action_dim - 1)
        else:
            state = torch.FloatTensor(state).unsqueeze(0)
            with torch.no_grad():
                q_values = self.online_net(state)
                return q_values.argmax().item()

    def store_transition(self, state, action, reward, next_state, done):
        self.memory.push(state, action, reward, next_state, done)

    def update(self):
        if len(self.memory) < BATCH_SIZE:
            return
        states, actions, rewards, next_states, dones = self.memory.sample(BATCH_SIZE)
        states = torch.FloatTensor(states)
        actions = torch.LongTensor(actions).unsqueeze(1)
        rewards = torch.FloatTensor(rewards).unsqueeze(1)
        next_states = torch.FloatTensor(next_states)
        dones = torch.FloatTensor(dones).unsqueeze(1)

        q_values = self.online_net(states).gather(1, actions)

        with torch.no_grad():
            next_actions = self.online_net(next_states).argmax(dim=1, keepdim=True)
            next_q_values = self.target_net(next_states).gather(1, next_actions)
            target_q_values = rewards + (1 - dones) * GAMMA * next_q_values

        loss = nn.MSELoss()(q_values, target_q_values)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

    def update_target_network(self):
        self.target_net.load_state_dict(self.online_net.state_dict())

注意 update 方法中，next_actions 是由在线网络选出的，而 next_q_values 是由目标网络评估的，这正是 DDQN 的关键。

训练循环

import gym
env = gym.make('CartPole-v1')
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = DoubleDQNAgent(state_dim, action_dim)

num_episodes = 500
epsilon_start = 1.0
epsilon_end = 0.01
epsilon_decay = 500

for episode in range(num_episodes):
    state = env.reset()
    done = False
    total_reward = 0
    while not done:
        epsilon = epsilon_end + (epsilon_start - epsilon_end) * np.exp(-1. * agent.steps_done / epsilon_decay)
        action = agent.select_action(state, epsilon)
        next_state, reward, done, _ = env.step(action)
        total_reward += reward
        agent.store_transition(state, action, reward, next_state, done)
        agent.update()
        state = next_state
        agent.steps_done += 1

    if episode % TARGET_UPDATE == 0:
        agent.update_target_network()
    print(f"Episode {episode}, Total Reward: {total_reward}")

env.close()

环境配置建议使用 Python 3.11.5, torch 2.1.0, gym 0.26.2。需要注意的是，本文代码主要用于展示算法逻辑，若应用于实际项目，通常还需要针对具体环境进行超参数调优。

优势与特点

1. 减小过估计偏差：分离动作选择和目标计算后，有效减少了过高估计的风险。
2. 更稳定的训练过程：估计值更准确，训练平滑，收敛速度更快。
3. 简单易实现：在 DQN 基础上仅需引入动作选择的分离逻辑。

总结

Double DQN 算法的提出主要是为了解决 DQN 中的'过估计偏差'问题。通过引入双网络，让动作选择和价值评估分离，大大提高了算法的稳定性和准确性。在实际工程中，这是处理离散动作空间强化学习任务的经典基线模型之一。