Double DQN 算法原理与 PyTorch 实战

公式推导

DDQN 的关键改进在于目标值的构造。传统 DQN 的 max 操作会导致过估计，而 DDQN 通过分离网络降低了这一风险。

DDQN 目标： $$y_t^{DDQN} = r_t + \gamma Q_{\theta^-}(s_{t+1}, \arg\max_a Q_{\theta}(s_{t+1}, a))$$

Q 学习目标： $$L(\theta) = \frac{1}{N} \sum_i \big(y_i - Q_{\theta}(s_i, a_i)\big)^2$$

动作 $a$ 由在线网络 $Q_{\theta}$ 选择，Q 值由目标网络 $Q_{\theta^-}$ 计算。这种设计确保了目标值的相对无偏性。

Python 实现

下面给出基于 PyTorch 的 Double DQN 完整实现，包含核心的双网络更新机制。

1. 导入必要库

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import random
from collections import deque

2. 超参数设置

GAMMA = 0.99          # 折扣因子
LR = 0.001            # 学习率
BATCH_SIZE = 64       # 批量大小
MEMORY_CAPACITY = 10000  # 经验池容量
TARGET_UPDATE = 10    # 目标网络更新周期

3. 定义网络结构

class QNetwork(nn.Module):
    def __init__(self, state_dim, action_dim):
        super(QNetwork, self).__init__()
        self.fc1 = nn.Linear(state_dim, 128)
        self.fc2 = nn.Linear(128, 128)
        self.fc3 = nn.Linear(128, action_dim)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

4. 经验回放池

class ReplayBuffer:
    def __init__(self, capacity):
        self.buffer = deque(maxlen=capacity)

    def push(self, state, action, reward, next_state, done):
        self.buffer.append((state, action, reward, next_state, done))

    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        states, actions, rewards, next_states, dones = zip(*batch)
        return (np.array(states), np.array(actions), np.array(rewards), 
                np.array(next_states), np.array(dones))

    def __len__(self):
        return len(self.buffer)

5. Double DQN Agent

class DoubleDQNAgent:
    def __init__(self, state_dim, action_dim):
        self.state_dim = state_dim
        self.action_dim = action_dim
        self.online_net = QNetwork(state_dim, action_dim)
        self.target_net = QNetwork(state_dim, action_dim)
        self.target_net.load_state_dict(self.online_net.state_dict())
        self.target_net.eval()
        self.optimizer = optim.Adam(self.online_net.parameters(), lr=LR)
        self.memory = ReplayBuffer(MEMORY_CAPACITY)
        self.steps_done = 0

    def select_action(self, state, epsilon):
        if random.random() < epsilon:
            return random.randint(0, self.action_dim - 1)
        else:
            state = torch.FloatTensor(state).unsqueeze(0)
            with torch.no_grad():
                q_values = self.online_net(state)
                return q_values.argmax().item()

    def store_transition(self, state, action, reward, next_state, done):
        self.memory.push(state, action, reward, next_state, done)

    def update(self):
        if len(self.memory) < BATCH_SIZE:
            return
        states, actions, rewards, next_states, dones = self.memory.sample(BATCH_SIZE)
        states = torch.FloatTensor(states)
        actions = torch.LongTensor(actions).unsqueeze(1)
        rewards = torch.FloatTensor(rewards).unsqueeze(1)
        next_states = torch.FloatTensor(next_states)
        dones = torch.FloatTensor(dones).unsqueeze(1)

        q_values = self.online_net(states).gather(1, actions)

        with torch.no_grad():
            next_actions = self.online_net(next_states).argmax(dim=1, keepdim=True)
            next_q_values = self.target_net(next_states).gather(1, next_actions)
            target_q_values = rewards + (1 - dones) * GAMMA * next_q_values

        loss = nn.MSELoss()(q_values, target_q_values)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

    def update_target_network(self):
        self.target_net.load_state_dict(self.online_net.state_dict())

6. 训练循环

import gym

env = gym.make('CartPole-v1')
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = DoubleDQNAgent(state_dim, action_dim)

num_episodes = 500
epsilon_start = 1.0
epsilon_end = 0.01
epsilon_decay = 500

for episode in range(num_episodes):
    state = env.reset()
    done = False
    total_reward = 0
    while not done:
        epsilon = epsilon_end + (epsilon_start - epsilon_end) * np.exp(-1. * agent.steps_done / epsilon_decay)
        action = agent.select_action(state, epsilon)
        next_state, reward, done, _ = env.step(action)
        total_reward += reward
        agent.store_transition(state, action, reward, next_state, done)
        agent.update()
        state = next_state
        agent.steps_done += 1

    if episode % TARGET_UPDATE == 0:
        agent.update_target_network()
    print(f"Episode {episode}, Total Reward: {total_reward}")

env.close()

优势与特点

1. 减小过估计偏差：分离机制有效降低了过高估计的风险。
2. 更稳定的训练过程：估计值更准确，收敛更快。
3. 简单易实现：在 DQN 基础上仅需调整动作选择逻辑。

总结

Double DQN 针对 DQN 的过估计偏差问题，通过引入双网络机制，让动作选择和价值评估分离，大幅提升了算法的稳定性和准确性。在实际工程中，它是处理复杂强化学习任务时的首选基线方案之一。