Initial commit: add project materials and code

code/baselines/__init__.py

@@ -0,0 +1,12 @@
+from .pure_coop import PureCooperative
+from .pure_comp import PureCompetitive
+from .single_dqn import SingleAgentDQN
+from .iddpg import IndependentDDPG
+from .fixed_lambda import FixedLambda
+from .equal_alloc import EqualAllocation
+from .semantic_only import SemanticOnly
+
+__all__ = [
+    "PureCooperative", "PureCompetitive", "SingleAgentDQN",
+    "IndependentDDPG", "FixedLambda", "EqualAllocation", "SemanticOnly",
+]

code/baselines/equal_alloc.py

@@ -0,0 +1,101 @@
+import numpy as np
+
+"""
+Baseline: EqualAllocation (等额分配基线)
+=====================================
+Purpose (lower bound): 
+- This baseline represents a simple heuristic approach with no learning involved.
+- It serves as a lower bound for performance comparison, showing the system behavior under a naive, fixed resource allocation strategy.
+- 目的（性能下限）：该基线代表了一种不涉及学习的简单启发式方法。它作为性能对比的下限，展示了在朴素的固定资源分配策略下系统的表现。
+
+Difference from Co-MADDPG:
+1. Learning: No learning vs Deep Reinforcement Learning.
+2. Action Selection: Always fixed at [0.5, 0.5, 0.5] for all resource parameters (subcarrier fraction, power, m_param).
+3. 与 Co-MADDPG 的区别：
+   - 学习机制：无学习 vs 深度强化学习。
+   - 动作选择：所有资源参数（子载波比例、功率、m 参数）始终固定为 [0.5, 0.5, 0.5]。
+
+Contribution:
+- Contributes to performance baseline tables as the "Random/Fixed" comparison point.
+- 贡献：作为“随机/固定”对比点，用于性能基准表。
+"""
+
+class DummyBuffer:
+    """
+    Dummy replay buffer that satisfies train.py's push/len interface.
+    满足 train.py 中 push/len 接口要求的虚拟重放池。
+    """
+    def push(self, *args):
+        # Do nothing as no learning is performed
+        # 不执行任何操作，因为没有学习过程
+        pass
+
+    def __len__(self):
+        # Always return 0 to indicate no samples available
+        # 始终返回 0，表示没有可用样本
+        return 0
+
+
+class EqualAllocation:
+    """
+    EqualAllocation algorithm implementation.
+    等额分配算法实现。
+    """
+    def __init__(self, config):
+        # Initialize with configuration and a dummy buffer
+        # 使用配置和虚拟重放池进行初始化
+        self.config = config
+        self.replay_buffer = DummyBuffer()
+
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Always return a fixed action [0.5, 0.5, 0.5].
+        始终返回固定动作 [0.5, 0.5, 0.5]。
+        """
+        return np.array([0.5, 0.5, 0.5], dtype=np.float32), \
+               np.array([0.5, 0.5, 0.5], dtype=np.float32)
+
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards using a fixed λ=0.5 for consistency in monitoring.
+        使用固定 λ=0.5 计算奖励，以保持监测的一致性。
+        
+        Formula: Balanced combination of coop and comp components.
+        公式说明：协作项与竞争项的平衡组合。
+        """
+        lam = 0.5
+        rew_cfg = self.config.get('reward', {})
+        coop_self = rew_cfg.get('coop_self', 0.5)
+        coop_other = rew_cfg.get('coop_other', 0.3)
+        coop_sys = rew_cfg.get('coop_sys', 0.2)
+        comp_self = rew_cfg.get('comp_self', 0.8)
+        comp_sys = rew_cfg.get('comp_sys', 0.2)
+
+        # Compute reward components for S
+        # 计算 S 的奖励组成部分
+        r_coop_s = coop_self * qoe_s + coop_other * qoe_b + coop_sys * qoe_sys
+        r_comp_s = comp_self * qoe_s + comp_sys * qoe_sys
+        r_s = lam * r_coop_s + (1 - lam) * r_comp_s
+
+        # Compute reward components for B
+        # 计算 B 的奖励组成部分
+        r_coop_b = coop_self * qoe_b + coop_other * qoe_s + coop_sys * qoe_sys
+        r_comp_b = comp_self * qoe_b + comp_sys * qoe_sys
+        r_b = lam * r_coop_b + (1 - lam) * r_comp_b
+
+        return r_s, r_b, lam
+
+    def update(self):
+        """
+        No update performed in heuristic baseline.
+        启发式基线中不执行更新。
+        """
+        return None
+
+    def save(self, path):
+        """No state to save."""
+        pass
+
+    def load(self, path):
+        """No state to load."""
+        pass

code/baselines/fixed_lambda.py

@@ -0,0 +1,280 @@
+import os
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+from agents.actor import Actor
+from agents.critic import Critic
+from agents.replay_buffer import ReplayBuffer
+from agents.noise import OUNoise
+
+"""
+Baseline: FixedLambda (固定 λ 基线)
+=====================================
+Purpose (ablation): 
+- This baseline is used to evaluate the benefit of the dynamic lambda switching mechanism in Co-MADDPG.
+- It fixes λ at a constant value (0.5), balancing cooperation and competition equally throughout the training.
+- 目的（消融实验）：该基线用于评估 Co-MADDPG 中动态 λ 切换机制的收益。它将 λ 固定为常数（0.5），在整个训练过程中平衡协作与竞争。
+
+Difference from Co-MADDPG:
+1. Lambda (λ): Fixed at 0.5, whereas Co-MADDPG dynamically adjusts λ based on system state.
+2. Update Order: Retains the Stackelberg update order (follower B first, then leader S), same as Co-MADDPG.
+3. 与 Co-MADDPG 的区别：
+   - Lambda (λ): 固定为 0.5，而 Co-MADDPG 根据系统状态动态调整 λ。
+   - 更新顺序：保留了 Stackelberg 博弈更新顺序（先更新从属者 B，再更新主导者 S），与 Co-MADDPG 一致。
+
+Contribution:
+- Contributes to performance sensitivity analysis regarding the choice of λ and shows why a fixed balance is suboptimal.
+- 贡献：用于关于 λ 选择的性能敏感性分析，展示为什么固定比例的平衡并非最优。
+"""
+
+class FixedLambda:
+    """
+    FixedLambda algorithm implementation.
+    固定 λ 算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # Hyperparameters: Gamma, Tau, Batch Size, and Fixed λ=0.5
+        # 超参数：折扣因子、软更新系数、批量大小以及固定 λ=0.5
+        self.gamma = config['training']['gamma']
+        self.tau = config['training']['tau']
+        self.batch_size = config['training']['batch_size']
+        self.fixed_lambda = 0.5
+
+        # Dimensions: State and Action
+        # 维度信息：状态与动作
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.act_dim = 3
+
+        # Actor networks and their target networks
+        # Actor 网络及其目标网络
+        hidden_a = config['network']['actor_hidden']
+        hidden_c = config['network']['critic_hidden']
+
+        self.actor_s = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_b = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_s_target = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_b_target = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+
+        # Joint Critics for Centralized Training
+        # 用于中心化训练的联合 Critic
+        obs_total = self.obs_dim * 2
+        act_total = self.act_dim * 2
+        self.critic_s = Critic(obs_total, act_total, hidden_c).to(self.device)
+        self.critic_b = Critic(obs_total, act_total, hidden_c).to(self.device)
+        self.critic_s_target = Critic(obs_total, act_total, hidden_c).to(self.device)
+        self.critic_b_target = Critic(obs_total, act_total, hidden_c).to(self.device)
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())
+
+        # Optimizers for actors and critics
+        # Actor 与 Critic 的优化器
+        self.actor_s_optimizer = torch.optim.Adam(self.actor_s.parameters(), lr=config['training']['actor_lr'])
+        self.actor_b_optimizer = torch.optim.Adam(self.actor_b.parameters(), lr=config['training']['actor_lr'])
+        self.critic_s_optimizer = torch.optim.Adam(self.critic_s.parameters(), lr=config['training']['critic_lr'])
+        self.critic_b_optimizer = torch.optim.Adam(self.critic_b.parameters(), lr=config['training']['critic_lr'])
+
+        # Experience Replay and OU Noise for exploration
+        # 经验重放池与用于探索的 OU 噪声
+        self.replay_buffer = ReplayBuffer(config['training']['buffer_capacity'])
+        self.noise_s = OUNoise(self.act_dim, theta=config['training']['ou_theta'],
+                               sigma_init=config['training']['ou_sigma_init'],
+                               sigma_min=config['training']['ou_sigma_min'])
+        self.noise_b = OUNoise(self.act_dim, theta=config['training']['ou_theta'],
+                               sigma_init=config['training']['ou_sigma_init'],
+                               sigma_min=config['training']['ou_sigma_min'])
+
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select actions for both agents given observations.
+        根据观察结果为两个智能体选择动作。
+        """
+        self.actor_s.eval()
+        self.actor_b.eval()
+        with torch.no_grad():
+            obs_s_t = torch.FloatTensor(obs_s).unsqueeze(0).to(self.device)
+            obs_b_t = torch.FloatTensor(obs_b).unsqueeze(0).to(self.device)
+            act_s = self.actor_s(obs_s_t).cpu().numpy()[0]
+            act_b = self.actor_b(obs_b_t).cpu().numpy()[0]
+        self.actor_s.train()
+        self.actor_b.train()
+
+        if explore:
+            # Add noise during training exploration
+            # 训练探索期间增加噪声
+            act_s = np.clip(act_s + self.noise_s.sample(), 0.0, 1.0)
+            act_b = np.clip(act_b + self.noise_b.sample(), 0.0, 1.0)
+        else:
+            act_s = np.clip(act_s, 0.0, 1.0)
+            act_b = np.clip(act_b, 0.0, 1.0)
+
+        return act_s, act_b
+
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards with fixed λ=0.5.
+        使用固定 λ=0.5 计算奖励。
+        
+        Formula: r_i = 0.5 * r_coop + 0.5 * r_comp
+        公式说明：奖励是协作项与竞争项的等权之和。
+        """
+        lam = self.fixed_lambda
+        rew_cfg = self.config.get('reward', {})
+        coop_self = rew_cfg.get('coop_self', 0.5)
+        coop_other = rew_cfg.get('coop_other', 0.3)
+        coop_sys = rew_cfg.get('coop_sys', 0.2)
+        comp_self = rew_cfg.get('comp_self', 0.8)
+        comp_sys = rew_cfg.get('comp_sys', 0.2)
+
+        # Compute Cooperative and Competitive components for S
+        # 计算 S 的协作与竞争组成部分
+        r_coop_s = coop_self * qoe_s + coop_other * qoe_b + coop_sys * qoe_sys
+        r_comp_s = comp_self * qoe_s + comp_sys * qoe_sys
+        r_s = lam * r_coop_s + (1 - lam) * r_comp_s
+
+        # Compute Cooperative and Competitive components for B
+        # 计算 B 的协作与竞争组成部分
+        r_coop_b = coop_self * qoe_b + coop_other * qoe_s + coop_sys * qoe_sys
+        r_comp_b = comp_self * qoe_b + comp_sys * qoe_sys
+        r_b = lam * r_coop_b + (1 - lam) * r_comp_b
+
+        return r_s, r_b, lam
+
+    def update(self):
+        """
+        Update networks using Stackelberg update order.
+        使用 Stackelberg 博弈顺序更新网络。
+        
+        Order: Follower B updates first, then Leader S updates considering B's response.
+        顺序：从属者 B 先更新，随后主导者 S 在考虑 B 的响应后进行更新。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+
+        # Sample from replay buffer
+        # 从经验池采样
+        obs_s, obs_b, act_s, act_b, rew_s, rew_b, next_obs_s, next_obs_b, dones = \
+            self.replay_buffer.sample(self.batch_size)
+
+        # Convert to tensors
+        # 转换为张量
+        obs_s = torch.FloatTensor(obs_s).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).to(self.device)
+        act_s = torch.FloatTensor(act_s).to(self.device)
+        act_b = torch.FloatTensor(act_b).to(self.device)
+        rew_s = torch.FloatTensor(rew_s).unsqueeze(1).to(self.device)
+        rew_b = torch.FloatTensor(rew_b).unsqueeze(1).to(self.device)
+        next_obs_s = torch.FloatTensor(next_obs_s).to(self.device)
+        next_obs_b = torch.FloatTensor(next_obs_b).to(self.device)
+        dones = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+
+        # Centralized observation and next observation
+        # 中心化观察与下一状态观察
+        joint_obs = torch.cat([obs_s, obs_b], dim=1)
+        joint_next_obs = torch.cat([next_obs_s, next_obs_b], dim=1)
+        joint_act = torch.cat([act_s, act_b], dim=1)
+
+        # Compute targets for critics
+        # 计算 Critic 的目标值
+        with torch.no_grad():
+            next_act_s = self.actor_s_target(next_obs_s)
+            next_act_b = self.actor_b_target(next_obs_b)
+            joint_next_act = torch.cat([next_act_s, next_act_b], dim=1)
+            target_q_s = rew_s + self.gamma * (1 - dones) * self.critic_s_target(joint_next_obs, joint_next_act)
+            target_q_b = rew_b + self.gamma * (1 - dones) * self.critic_b_target(joint_next_obs, joint_next_act)
+
+        # --- Stackelberg: update follower B first ---
+        # --- Stackelberg 博弈：首先更新从属者 B ---
+        
+        # Update Critic B
+        # 更新 Critic B
+        current_q_b = self.critic_b(joint_obs, joint_act)
+        critic_loss_b = F.mse_loss(current_q_b, target_q_b)
+        self.critic_b_optimizer.zero_grad()
+        critic_loss_b.backward()
+        self.critic_b_optimizer.step()
+
+        # Update Actor B (Follower)
+        # 更新 Actor B (从属者)
+        new_act_b = self.actor_b(obs_b)
+        actor_loss_b = -self.critic_b(joint_obs, torch.cat([act_s, new_act_b], dim=1)).mean()
+        self.actor_b_optimizer.zero_grad()
+        actor_loss_b.backward()
+        self.actor_b_optimizer.step()
+
+        # --- Then update leader S ---
+        # --- 然后更新主导者 S ---
+        
+        # Re-compute follower's best response for leader's critic update
+        # 为主导者的 Critic 更新重新计算从属者的最佳响应
+        with torch.no_grad():
+            act_b_br = self.actor_b(obs_b)
+        joint_act_leader = torch.cat([act_s, act_b_br], dim=1)
+
+        # Update Critic S
+        # 更新 Critic S
+        current_q_s = self.critic_s(joint_obs, joint_act_leader)
+        critic_loss_s = F.mse_loss(current_q_s, target_q_s)
+        self.critic_s_optimizer.zero_grad()
+        critic_loss_s.backward()
+        self.critic_s_optimizer.step()
+
+        # Update Actor S (Leader) considering Follower's best response
+        # 考虑从属者的最佳响应，更新 Actor S (主导者)
+        with torch.no_grad():
+            act_b_br2 = self.actor_b(obs_b)
+        new_act_s = self.actor_s(obs_s)
+        actor_loss_s = -self.critic_s(joint_obs, torch.cat([new_act_s, act_b_br2], dim=1)).mean()
+        self.actor_s_optimizer.zero_grad()
+        actor_loss_s.backward()
+        self.actor_s_optimizer.step()
+
+        # Soft update target networks
+        # 目标网络软更新
+        for target, source in [
+            (self.critic_s_target, self.critic_s),
+            (self.critic_b_target, self.critic_b),
+            (self.actor_s_target, self.actor_s),
+            (self.actor_b_target, self.actor_b),
+        ]:
+            for tp, sp in zip(target.parameters(), source.parameters()):
+                tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {
+            'actor_loss_s': actor_loss_s.item(),
+            'actor_loss_b': actor_loss_b.item(),
+            'critic_loss_s': critic_loss_s.item(),
+            'critic_loss_b': critic_loss_b.item(),
+        }
+
+    def save(self, path):
+        """
+        Save models to disk.
+        将模型保存至磁盘。
+        """
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.actor_s.state_dict(), os.path.join(path, "actor_s.pth"))
+        torch.save(self.actor_b.state_dict(), os.path.join(path, "actor_b.pth"))
+        torch.save(self.critic_s.state_dict(), os.path.join(path, "critic_s.pth"))
+        torch.save(self.critic_b.state_dict(), os.path.join(path, "critic_b.pth"))
+
+    def load(self, path):
+        """
+        Load models from disk.
+        从磁盘加载模型。
+        """
+        self.actor_s.load_state_dict(torch.load(os.path.join(path, "actor_s.pth"), map_location=self.device))
+        self.actor_b.load_state_dict(torch.load(os.path.join(path, "actor_b.pth"), map_location=self.device))
+        self.critic_s.load_state_dict(torch.load(os.path.join(path, "critic_s.pth"), map_location=self.device))
+        self.critic_b.load_state_dict(torch.load(os.path.join(path, "critic_b.pth"), map_location=self.device))
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())

code/baselines/iddpg.py

@@ -0,0 +1,266 @@
+import os
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from agents.actor import Actor
+from agents.replay_buffer import ReplayBuffer
+from agents.noise import OUNoise
+
+"""
+Baseline: IndependentDDPG (独立 DDPG 基线)
+=====================================
+Purpose (ablation): 
+- This baseline removes the Centralized Training Decentralized Execution (CTDE) component.
+- It is used to demonstrate the necessity of joint critics that observe other agents' actions for stable training in MARL.
+- 目的（消融实验）：该基线移除了中心化训练分布式执行（CTDE）组件。用于证明在多智能体强化学习中，引入能观察其他智能体动作的联合 Critic 对维持训练稳定性的必要性。
+
+Difference from Co-MADDPG:
+1. Critic Type: IndependentCritics are used, which only take the local observation and local action (obs_i, act_i) as input.
+2. Update Order: Simultaneous independent updates for both agents.
+3. 与 Co-MADDPG 的区别：
+   - Critic 类型：使用独立 Critic，其输入仅包含局部观察与局部动作 (obs_i, act_i)。
+   - 更新顺序：两个智能体同时进行独立的更新。
+
+Contribution:
+- Contributes to ablation studies showing how centralized critics mitigate non-stationarity issues.
+- 贡献：用于消融实验，展示中心化 Critic 如何缓解非平稳性（Non-stationarity）问题。
+"""
+
+class IndependentCritic(nn.Module):
+    """
+    IndependentCritic that takes only a single agent's observation and action.
+    独立 Critic，仅接收单个智能体的观察与动作。
+    """
+    def __init__(self, obs_dim, act_dim, hidden_sizes=[512, 512, 256]):
+        super().__init__()
+        assert len(hidden_sizes) == 3
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim + act_dim, hidden_sizes[0]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[0], hidden_sizes[1]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[1], hidden_sizes[2]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[2], 1),
+        )
+
+    def forward(self, obs, act):
+        # Concatenate local observation and local action
+        # 拼接局部观察与局部动作
+        x = torch.cat([obs, act], dim=1)
+        return self.net(x)
+
+
+class IndependentDDPG:
+    """
+    IndependentDDPG algorithm implementation.
+    独立 DDPG 算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # Hyperparameters
+        # 超参数
+        self.gamma = config['training']['gamma']
+        self.tau = config['training']['tau']
+        self.batch_size = config['training']['batch_size']
+
+        # Dimensions
+        # 维度
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.act_dim = 3
+
+        # Hidden layer configurations
+        # 隐藏层配置
+        hidden_a = config['network']['actor_hidden']
+        hidden_c = config['network']['critic_hidden']
+
+        # Agent S: Local Actor and Independent Critic
+        # 智能体 S：局部 Actor 与独立 Critic
+        self.actor_s = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_s_target = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.critic_s = IndependentCritic(self.obs_dim, self.act_dim, hidden_c).to(self.device)
+        self.critic_s_target = IndependentCritic(self.obs_dim, self.act_dim, hidden_c).to(self.device)
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+
+        # Agent B: Local Actor and Independent Critic
+        # 智能体 B：局部 Actor 与独立 Critic
+        self.actor_b = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_b_target = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        self.critic_b = IndependentCritic(self.obs_dim, self.act_dim, hidden_c).to(self.device)
+        self.critic_b_target = IndependentCritic(self.obs_dim, self.act_dim, hidden_c).to(self.device)
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())
+
+        # Optimizers
+        # 优化器
+        self.actor_s_optimizer = torch.optim.Adam(self.actor_s.parameters(), lr=config['training']['actor_lr'])
+        self.actor_b_optimizer = torch.optim.Adam(self.actor_b.parameters(), lr=config['training']['actor_lr'])
+        self.critic_s_optimizer = torch.optim.Adam(self.critic_s.parameters(), lr=config['training']['critic_lr'])
+        self.critic_b_optimizer = torch.optim.Adam(self.critic_b.parameters(), lr=config['training']['critic_lr'])
+
+        # Shared replay buffer
+        # 共享重放池
+        self.replay_buffer = ReplayBuffer(config['training']['buffer_capacity'])
+
+        # Noise for exploration
+        # 探索噪声
+        self.noise_s = OUNoise(self.act_dim, theta=config['training']['ou_theta'],
+                               sigma_init=config['training']['ou_sigma_init'],
+                               sigma_min=config['training']['ou_sigma_min'])
+        self.noise_b = OUNoise(self.act_dim, theta=config['training']['ou_theta'],
+                               sigma_init=config['training']['ou_sigma_init'],
+                               sigma_min=config['training']['ou_sigma_min'])
+
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select actions for both agents.
+        为两个智能体选择动作。
+        """
+        self.actor_s.eval()
+        self.actor_b.eval()
+        with torch.no_grad():
+            obs_s_t = torch.FloatTensor(obs_s).unsqueeze(0).to(self.device)
+            obs_b_t = torch.FloatTensor(obs_b).unsqueeze(0).to(self.device)
+            act_s = self.actor_s(obs_s_t).cpu().numpy()[0]
+            act_b = self.actor_b(obs_b_t).cpu().numpy()[0]
+        self.actor_s.train()
+        self.actor_b.train()
+
+        if explore:
+            # Apply OU noise
+            # 应用 OU 噪声
+            act_s = np.clip(act_s + self.noise_s.sample(), 0.0, 1.0)
+            act_b = np.clip(act_b + self.noise_b.sample(), 0.0, 1.0)
+        else:
+            act_s = np.clip(act_s, 0.0, 1.0)
+            act_b = np.clip(act_b, 0.0, 1.0)
+
+        return act_s, act_b
+
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards based on independent competitive behavior (λ=0).
+        基于独立的竞争行为计算奖励 (λ=0)。
+        
+        Formula: r_i = comp_self * qoe_i + comp_sys * qoe_sys
+        公式说明：独立模式下默认为纯竞争，每个智能体仅优化自身效用及系统整体惩罚。
+        """
+        lam = 0.0
+        r_s = self.config['reward']['comp_self'] * qoe_s + self.config['reward']['comp_sys'] * qoe_sys
+        r_b = self.config['reward']['comp_self'] * qoe_b + self.config['reward']['comp_sys'] * qoe_sys
+        return r_s, r_b, lam
+
+    def update(self):
+        """
+        Update each agent independently and simultaneously.
+        独立且同步地更新每个智能体。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+
+        # Sample batch
+        # 采样批量数据
+        obs_s, obs_b, act_s, act_b, rew_s, rew_b, next_obs_s, next_obs_b, dones = \
+            self.replay_buffer.sample(self.batch_size)
+
+        # To tensors
+        # 转换为张量
+        obs_s = torch.FloatTensor(obs_s).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).to(self.device)
+        act_s = torch.FloatTensor(act_s).to(self.device)
+        act_b = torch.FloatTensor(act_b).to(self.device)
+        rew_s = torch.FloatTensor(rew_s).unsqueeze(1).to(self.device)
+        rew_b = torch.FloatTensor(rew_b).unsqueeze(1).to(self.device)
+        next_obs_s = torch.FloatTensor(next_obs_s).to(self.device)
+        next_obs_b = torch.FloatTensor(next_obs_b).to(self.device)
+        dones = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+
+        # --- Update Agent S (independent) ---
+        # --- 独立更新智能体 S ---
+        with torch.no_grad():
+            # Critic target only uses local next observation and action
+            # Critic 目标仅使用局部下一状态观察与动作
+            next_act_s = self.actor_s_target(next_obs_s)
+            target_q_s = rew_s + self.gamma * (1 - dones) * self.critic_s_target(next_obs_s, next_act_s)
+        
+        current_q_s = self.critic_s(obs_s, act_s)
+        critic_loss_s = F.mse_loss(current_q_s, target_q_s)
+        self.critic_s_optimizer.zero_grad()
+        critic_loss_s.backward()
+        self.critic_s_optimizer.step()
+
+        new_act_s = self.actor_s(obs_s)
+        actor_loss_s = -self.critic_s(obs_s, new_act_s).mean()
+        self.actor_s_optimizer.zero_grad()
+        actor_loss_s.backward()
+        self.actor_s_optimizer.step()
+
+        # --- Update Agent B (independent) ---
+        # --- 独立更新智能体 B ---
+        with torch.no_grad():
+            # Critic target only uses local next observation and action
+            # Critic 目标仅使用局部下一状态观察与动作
+            next_act_b = self.actor_b_target(next_obs_b)
+            target_q_b = rew_b + self.gamma * (1 - dones) * self.critic_b_target(next_obs_b, next_act_b)
+        
+        current_q_b = self.critic_b(obs_b, act_b)
+        critic_loss_b = F.mse_loss(current_q_b, target_q_b)
+        self.critic_b_optimizer.zero_grad()
+        critic_loss_b.backward()
+        self.critic_b_optimizer.step()
+
+        new_act_b = self.actor_b(obs_b)
+        actor_loss_b = -self.critic_b(obs_b, new_act_b).mean()
+        self.actor_b_optimizer.zero_grad()
+        actor_loss_b.backward()
+        self.actor_b_optimizer.step()
+
+        # Soft update targets for both agents
+        # 软更新两个智能体的目标网络
+        for target, source in [
+            (self.critic_s_target, self.critic_s),
+            (self.critic_b_target, self.critic_b),
+            (self.actor_s_target, self.actor_s),
+            (self.actor_b_target, self.actor_b),
+        ]:
+            for tp, sp in zip(target.parameters(), source.parameters()):
+                tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {
+            'actor_loss_s': actor_loss_s.item(),
+            'actor_loss_b': actor_loss_b.item(),
+            'critic_loss_s': critic_loss_s.item(),
+            'critic_loss_b': critic_loss_b.item(),
+        }
+
+    def save(self, path):
+        """
+        Save models.
+        保存模型。
+        """
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.actor_s.state_dict(), os.path.join(path, "actor_s.pth"))
+        torch.save(self.actor_b.state_dict(), os.path.join(path, "actor_b.pth"))
+        torch.save(self.critic_s.state_dict(), os.path.join(path, "critic_s.pth"))
+        torch.save(self.critic_b.state_dict(), os.path.join(path, "critic_b.pth"))
+
+    def load(self, path):
+        """
+        Load models.
+        加载模型。
+        """
+        self.actor_s.load_state_dict(torch.load(os.path.join(path, "actor_s.pth"), map_location=self.device))
+        self.actor_b.load_state_dict(torch.load(os.path.join(path, "actor_b.pth"), map_location=self.device))
+        self.critic_s.load_state_dict(torch.load(os.path.join(path, "critic_s.pth"), map_location=self.device))
+        self.critic_b.load_state_dict(torch.load(os.path.join(path, "critic_b.pth"), map_location=self.device))
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())

code/baselines/pure_comp.py

@@ -0,0 +1,245 @@
+import os
+import torch
+import torch.nn.functional as F
+import numpy as np
+from agents.actor import Actor
+from agents.critic import Critic
+from agents.replay_buffer import ReplayBuffer
+from agents.noise import OUNoise
+
+"""
+Baseline: PureCompetitive (纯竞争基线)
+=====================================
+Purpose (ablation): 
+- This baseline removes the cooperative component from the MADDPG framework.
+- It serves as an ablation study to demonstrate that pure competition (λ=0) leads to resource wastage and suboptimal system-wide utility.
+- 目的（消融实验）：该基线移除了 MADDPG 框架中的协作成分。作为消融实验，用于证明纯竞争模式（λ=0）会导致资源浪费和系统级效用降低。
+
+Difference from Co-MADDPG:
+1. Lambda (λ): Fixed at 0.0 (pure competition), whereas Co-MADDPG uses dynamic λ.
+2. Update Order: Uses simultaneous updates for both actors, whereas Co-MADDPG uses Stackelberg update order.
+3. 与 Co-MADDPG 的区别：
+   - Lambda (λ): 固定为 0.0（纯竞争），而 Co-MADDPG 使用动态 λ。
+   - 更新顺序：两个参与者同时更新（Simultaneous Update），而 Co-MADDPG 使用 Stackelberg 博弈更新顺序。
+
+Contribution:
+- Contributes to comparison figures showing the "Price of Anarchy" in resource allocation.
+- 贡献：用于对比图表，展示资源分配中的“无政府代价”。
+"""
+
+class PureCompetitive:
+    """
+    PureCompetitive algorithm implementation.
+    纯竞争算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+        # Hyperparameters: Gamma (discount), Tau (soft update), Batch Size
+        # 超参数：折扣因子、软更新系数、批量大小
+        self.gamma = config['training']['gamma']
+        self.tau = config['training']['tau']
+        self.batch_size = config['training']['batch_size']
+        
+        # Dimensions: State (subcarriers + 4), Action (3)
+        # 维度：状态（子载波 + 4）、动作（3）
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.act_dim = 3
+        
+        # Agents: Semantic (s) and Traditional (b) actors and target networks
+        # 智能体：语义 (s) 与 传统 (b) 参与者的 Actor 及其目标网络
+        self.actor_s = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_b = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_s_target = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_b_target = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        
+        # Joint Critics: Uses Centralized Training (obs_dim*2, act_dim*2)
+        # 联合 Critic：使用中心化训练（输入为两体观察与动作的并集）
+        self.critic_s = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_b = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_s_target = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_b_target = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())
+        
+        # Optimizers for all networks
+        # 所有网络的优化器
+        self.actor_s_optimizer = torch.optim.Adam(self.actor_s.parameters(), lr=config['training']['actor_lr'])
+        self.actor_b_optimizer = torch.optim.Adam(self.actor_b.parameters(), lr=config['training']['actor_lr'])
+        self.critic_s_optimizer = torch.optim.Adam(self.critic_s.parameters(), lr=config['training']['critic_lr'])
+        self.critic_b_optimizer = torch.optim.Adam(self.critic_b.parameters(), lr=config['training']['critic_lr'])
+        
+        # Experience Replay and Noise for exploration
+        # 经验重放池与用于探索的噪声
+        self.replay_buffer = ReplayBuffer(config['training']['buffer_capacity'])
+        self.noise_s = OUNoise(self.act_dim, theta=config['training']['ou_theta'], sigma_init=config['training']['ou_sigma_init'], sigma_min=config['training']['ou_sigma_min'])
+        self.noise_b = OUNoise(self.act_dim, theta=config['training']['ou_theta'], sigma_init=config['training']['ou_sigma_init'], sigma_min=config['training']['ou_sigma_min'])
+        
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select actions for both agents given observations.
+        根据观察结果为两个智能体选择动作。
+        """
+        obs_s = torch.FloatTensor(obs_s).unsqueeze(0).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).unsqueeze(0).to(self.device)
+        
+        self.actor_s.eval()
+        self.actor_b.eval()
+        with torch.no_grad():
+            # Forward pass through actors
+            # Actor 前向传播
+            act_s = self.actor_s(obs_s).cpu().numpy()[0]
+            act_b = self.actor_b(obs_b).cpu().numpy()[0]
+        self.actor_s.train()
+        self.actor_b.train()
+        
+        if explore:
+            # Apply OU noise for exploration
+            # 应用 OU 噪声进行探索
+            act_s = np.clip(act_s + self.noise_s.sample(), 0.0, 1.0)
+            act_b = np.clip(act_b + self.noise_b.sample(), 0.0, 1.0)
+            
+        return act_s, act_b
+        
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards based on pure competition (λ=0).
+        基于纯竞争计算奖励 (λ=0)。
+        
+        Formula: r_i = comp_self * qoe_i + comp_sys * qoe_sys
+        公式说明：由于 λ=0，奖励完全由竞争项组成，仅考虑自身 QoE 以及系统总 QoE 的惩罚项。
+        """
+        lam = 0.0
+        r_s = self.config['reward']['comp_self'] * qoe_s + self.config['reward']['comp_sys'] * qoe_sys
+        r_b = self.config['reward']['comp_self'] * qoe_b + self.config['reward']['comp_sys'] * qoe_sys
+        return r_s, r_b, lam
+        
+    def update(self):
+        """
+        Update the networks using sampled experiences.
+        使用采样的经验更新网络。
+        
+        Update order: Simultaneous updates (both actors update based on current policy of the other).
+        更新顺序：同时更新（两个 Actor 基于对方当前的策略进行更新）。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+            
+        # Sample batch from replay buffer
+        # 从重放池采样批量数据
+        obs_s, obs_b, act_s, act_b, rew_s, rew_b, next_obs_s, next_obs_b, dones = self.replay_buffer.sample(self.batch_size)
+        
+        # Convert to tensors
+        # 转换为张量
+        obs_s = torch.FloatTensor(obs_s).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).to(self.device)
+        act_s = torch.FloatTensor(act_s).to(self.device)
+        act_b = torch.FloatTensor(act_b).to(self.device)
+        rew_s = torch.FloatTensor(rew_s).unsqueeze(1).to(self.device)
+        rew_b = torch.FloatTensor(rew_b).unsqueeze(1).to(self.device)
+        next_obs_s = torch.FloatTensor(next_obs_s).to(self.device)
+        next_obs_b = torch.FloatTensor(next_obs_b).to(self.device)
+        dones = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+        
+        # Centralized observations and actions
+        # 中心化观察与动作
+        joint_obs = torch.cat([obs_s, obs_b], dim=1)
+        joint_next_obs = torch.cat([next_obs_s, next_obs_b], dim=1)
+        joint_act = torch.cat([act_s, act_b], dim=1)
+        
+        # 1. Critics Update (1. Critic 更新)
+        with torch.no_grad():
+            # Get target actions for next state
+            # 获取下一状态的目标动作
+            next_act_s = self.actor_s_target(next_obs_s)
+            next_act_b = self.actor_b_target(next_obs_b)
+            joint_next_act = torch.cat([next_act_s, next_act_b], dim=1)
+            
+            # Compute target Q values
+            # 计算目标 Q 值
+            target_q_s = rew_s + self.gamma * (1 - dones) * self.critic_s_target(joint_next_obs, joint_next_act)
+            target_q_b = rew_b + self.gamma * (1 - dones) * self.critic_b_target(joint_next_obs, joint_next_act)
+            
+        # Compute current Q values and MSE loss
+        # 计算当前 Q 值与均方误差损失
+        current_q_s = self.critic_s(joint_obs, joint_act)
+        current_q_b = self.critic_b(joint_obs, joint_act)
+        
+        critic_loss_s = F.mse_loss(current_q_s, target_q_s)
+        critic_loss_b = F.mse_loss(current_q_b, target_q_b)
+        
+        # Backpropagation for critics
+        # Critic 的反向传播
+        self.critic_s_optimizer.zero_grad()
+        critic_loss_s.backward()
+        self.critic_s_optimizer.step()
+        
+        self.critic_b_optimizer.zero_grad()
+        critic_loss_b.backward()
+        self.critic_b_optimizer.step()
+        
+        # 2. Actors Update (Simultaneous) (2. Actor 更新 - 同时进行)
+        new_act_s = self.actor_s(obs_s)
+        new_act_b = self.actor_b(obs_b)
+        
+        # Calculate policy loss using joint critic
+        # 使用联合 Critic 计算策略损失
+        actor_loss_s = -self.critic_s(joint_obs, torch.cat([new_act_s, act_b], dim=1)).mean()
+        actor_loss_b = -self.critic_b(joint_obs, torch.cat([act_s, new_act_b], dim=1)).mean()
+        
+        # Backpropagation for actors
+        # Actor 的反向传播
+        self.actor_s_optimizer.zero_grad()
+        actor_loss_s.backward()
+        self.actor_s_optimizer.step()
+        
+        self.actor_b_optimizer.zero_grad()
+        actor_loss_b.backward()
+        self.actor_b_optimizer.step()
+        
+        # 3. Soft Target Networks Update (3. 目标网络软更新)
+        for target_param, param in zip(self.critic_s_target.parameters(), self.critic_s.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.critic_b_target.parameters(), self.critic_b.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.actor_s_target.parameters(), self.actor_s.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.actor_b_target.parameters(), self.actor_b.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+            
+        return {
+            'actor_loss_s': actor_loss_s.item(),
+            'actor_loss_b': actor_loss_b.item(),
+            'critic_loss_s': critic_loss_s.item(),
+            'critic_loss_b': critic_loss_b.item()
+        }
+
+    def save(self, path):
+        """
+        Save models to disk.
+        将模型保存至磁盘。
+        """
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.actor_s.state_dict(), os.path.join(path, "actor_s.pth"))
+        torch.save(self.actor_b.state_dict(), os.path.join(path, "actor_b.pth"))
+        torch.save(self.critic_s.state_dict(), os.path.join(path, "critic_s.pth"))
+        torch.save(self.critic_b.state_dict(), os.path.join(path, "critic_b.pth"))
+        
+    def load(self, path):
+        """
+        Load models from disk.
+        从磁盘加载模型。
+        """
+        self.actor_s.load_state_dict(torch.load(os.path.join(path, "actor_s.pth"), map_location=self.device))
+        self.actor_b.load_state_dict(torch.load(os.path.join(path, "actor_b.pth"), map_location=self.device))
+        self.critic_s.load_state_dict(torch.load(os.path.join(path, "critic_s.pth"), map_location=self.device))
+        self.critic_b.load_state_dict(torch.load(os.path.join(path, "critic_b.pth"), map_location=self.device))
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())

code/baselines/pure_coop.py

@@ -0,0 +1,245 @@
+import os
+import torch
+import torch.nn.functional as F
+import numpy as np
+from agents.actor import Actor
+from agents.critic import Critic
+from agents.replay_buffer import ReplayBuffer
+from agents.noise import OUNoise
+
+"""
+Baseline: PureCooperative (纯协作基线)
+=====================================
+Purpose (ablation): 
+- This baseline removes the competitive component from the MADDPG framework.
+- It serves as an ablation study to demonstrate the necessity of competitive modeling (λ < 1) for system performance.
+- 目的（消融实验）：该基线移除了 MADDPG 框架中的竞争成分。作为消融实验，用于证明在系统中引入竞争建模（λ < 1）对性能提升的必要性。
+
+Difference from Co-MADDPG:
+1. Lambda (λ): Fixed at 1.0 (pure cooperation), whereas Co-MADDPG uses dynamic λ.
+2. Update Order: Uses simultaneous updates for both actors, whereas Co-MADDPG uses Stackelberg update order.
+3. 与 Co-MADDPG 的区别：
+   - Lambda (λ): 固定为 1.0（纯协作），而 Co-MADDPG 使用动态 λ。
+   - 更新顺序：两个参与者同时更新（Simultaneous Update），而 Co-MADDPG 使用 Stackelberg 博弈更新顺序。
+
+Contribution:
+- Contributes to performance comparison figures and tables (e.g., convergence speed and final QoE) to show how pure cooperation handles resource conflicts.
+- 贡献：用于性能对比图表（如收敛速度和最终 QoE），展示纯协作模式在处理资源冲突时的表现。
+"""
+
+class PureCooperative:
+    """
+    PureCooperative algorithm implementation.
+    纯协作算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+        # Hyperparameters: Gamma (discount), Tau (soft update), Batch Size
+        # 超参数：折扣因子、软更新系数、批量大小
+        self.gamma = config['training']['gamma']
+        self.tau = config['training']['tau']
+        self.batch_size = config['training']['batch_size']
+        
+        # Dimensions: State (subcarriers + 4), Action (3)
+        # 维度：状态（子载波 + 4）、动作（3）
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.act_dim = 3
+        
+        # Agents: Semantic (s) and Traditional (b) actors and target networks
+        # 智能体：语义 (s) 与 传统 (b) 参与者的 Actor 及其目标网络
+        self.actor_s = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_b = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_s_target = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_b_target = Actor(self.obs_dim, self.act_dim, config['network']['actor_hidden']).to(self.device)
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        
+        # Joint Critics: Uses Centralized Training (obs_dim*2, act_dim*2)
+        # 联合 Critic：使用中心化训练（输入为两体观察与动作的并集）
+        self.critic_s = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_b = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_s_target = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_b_target = Critic(self.obs_dim*2, self.act_dim*2, config['network']['critic_hidden']).to(self.device)
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())
+        
+        # Optimizers for all networks
+        # 所有网络的优化器
+        self.actor_s_optimizer = torch.optim.Adam(self.actor_s.parameters(), lr=config['training']['actor_lr'])
+        self.actor_b_optimizer = torch.optim.Adam(self.actor_b.parameters(), lr=config['training']['actor_lr'])
+        self.critic_s_optimizer = torch.optim.Adam(self.critic_s.parameters(), lr=config['training']['critic_lr'])
+        self.critic_b_optimizer = torch.optim.Adam(self.critic_b.parameters(), lr=config['training']['critic_lr'])
+        
+        # Experience Replay and Noise for exploration
+        # 经验重放池与用于探索的噪声
+        self.replay_buffer = ReplayBuffer(config['training']['buffer_capacity'])
+        self.noise_s = OUNoise(self.act_dim, theta=config['training']['ou_theta'], sigma_init=config['training']['ou_sigma_init'], sigma_min=config['training']['ou_sigma_min'])
+        self.noise_b = OUNoise(self.act_dim, theta=config['training']['ou_theta'], sigma_init=config['training']['ou_sigma_init'], sigma_min=config['training']['ou_sigma_min'])
+        
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select actions for both agents given observations.
+        根据观察结果为两个智能体选择动作。
+        """
+        obs_s = torch.FloatTensor(obs_s).unsqueeze(0).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).unsqueeze(0).to(self.device)
+        
+        self.actor_s.eval()
+        self.actor_b.eval()
+        with torch.no_grad():
+            # Forward pass through actors
+            # Actor 前向传播
+            act_s = self.actor_s(obs_s).cpu().numpy()[0]
+            act_b = self.actor_b(obs_b).cpu().numpy()[0]
+        self.actor_s.train()
+        self.actor_b.train()
+        
+        if explore:
+            # Apply OU noise for exploration
+            # 应用 OU 噪声进行探索
+            act_s = np.clip(act_s + self.noise_s.sample(), 0.0, 1.0)
+            act_b = np.clip(act_b + self.noise_b.sample(), 0.0, 1.0)
+            
+        return act_s, act_b
+        
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards based on pure cooperation (λ=1).
+        基于纯协作计算奖励 (λ=1)。
+        
+        Formula: r_i = coop_self * qoe_i + coop_other * qoe_j + coop_sys * qoe_sys
+        公式说明：由于 λ=1，奖励完全由协作项组成，考虑自身 QoE、对方 QoE 以及系统总 QoE。
+        """
+        lam = 1.0
+        r_s = self.config['reward']['coop_self'] * qoe_s + self.config['reward']['coop_other'] * qoe_b + self.config['reward']['coop_sys'] * qoe_sys
+        r_b = self.config['reward']['coop_self'] * qoe_b + self.config['reward']['coop_other'] * qoe_s + self.config['reward']['coop_sys'] * qoe_sys
+        return r_s, r_b, lam
+        
+    def update(self):
+        """
+        Update the networks using sampled experiences.
+        使用采样的经验更新网络。
+        
+        Update order: Simultaneous updates (both actors update based on current policy of the other).
+        更新顺序：同时更新（两个 Actor 基于对方当前的策略进行更新）。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+            
+        # Sample batch from replay buffer
+        # 从重放池采样批量数据
+        obs_s, obs_b, act_s, act_b, rew_s, rew_b, next_obs_s, next_obs_b, dones = self.replay_buffer.sample(self.batch_size)
+        
+        # Convert to tensors
+        # 转换为张量
+        obs_s = torch.FloatTensor(obs_s).to(self.device)
+        obs_b = torch.FloatTensor(obs_b).to(self.device)
+        act_s = torch.FloatTensor(act_s).to(self.device)
+        act_b = torch.FloatTensor(act_b).to(self.device)
+        rew_s = torch.FloatTensor(rew_s).unsqueeze(1).to(self.device)
+        rew_b = torch.FloatTensor(rew_b).unsqueeze(1).to(self.device)
+        next_obs_s = torch.FloatTensor(next_obs_s).to(self.device)
+        next_obs_b = torch.FloatTensor(next_obs_b).to(self.device)
+        dones = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+        
+        # Centralized observations and actions
+        # 中心化观察与动作
+        joint_obs = torch.cat([obs_s, obs_b], dim=1)
+        joint_next_obs = torch.cat([next_obs_s, next_obs_b], dim=1)
+        joint_act = torch.cat([act_s, act_b], dim=1)
+        
+        # 1. Critics Update (1. Critic 更新)
+        with torch.no_grad():
+            # Get target actions for next state
+            # 获取下一状态的目标动作
+            next_act_s = self.actor_s_target(next_obs_s)
+            next_act_b = self.actor_b_target(next_obs_b)
+            joint_next_act = torch.cat([next_act_s, next_act_b], dim=1)
+            
+            # Compute target Q values
+            # 计算目标 Q 值
+            target_q_s = rew_s + self.gamma * (1 - dones) * self.critic_s_target(joint_next_obs, joint_next_act)
+            target_q_b = rew_b + self.gamma * (1 - dones) * self.critic_b_target(joint_next_obs, joint_next_act)
+            
+        # Compute current Q values and MSE loss
+        # 计算当前 Q 值与均方误差损失
+        current_q_s = self.critic_s(joint_obs, joint_act)
+        current_q_b = self.critic_b(joint_obs, joint_act)
+        
+        critic_loss_s = F.mse_loss(current_q_s, target_q_s)
+        critic_loss_b = F.mse_loss(current_q_b, target_q_b)
+        
+        # Backpropagation for critics
+        # Critic 的反向传播
+        self.critic_s_optimizer.zero_grad()
+        critic_loss_s.backward()
+        self.critic_s_optimizer.step()
+        
+        self.critic_b_optimizer.zero_grad()
+        critic_loss_b.backward()
+        self.critic_b_optimizer.step()
+        
+        # 2. Actors Update (Simultaneous) (2. Actor 更新 - 同时进行)
+        new_act_s = self.actor_s(obs_s)
+        new_act_b = self.actor_b(obs_b)
+        
+        # Calculate policy loss using joint critic
+        # 使用联合 Critic 计算策略损失
+        actor_loss_s = -self.critic_s(joint_obs, torch.cat([new_act_s, act_b], dim=1)).mean()
+        actor_loss_b = -self.critic_b(joint_obs, torch.cat([act_s, new_act_b], dim=1)).mean()
+        
+        # Backpropagation for actors
+        # Actor 的反向传播
+        self.actor_s_optimizer.zero_grad()
+        actor_loss_s.backward()
+        self.actor_s_optimizer.step()
+        
+        self.actor_b_optimizer.zero_grad()
+        actor_loss_b.backward()
+        self.actor_b_optimizer.step()
+        
+        # 3. Soft Target Networks Update (3. 目标网络软更新)
+        for target_param, param in zip(self.critic_s_target.parameters(), self.critic_s.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.critic_b_target.parameters(), self.critic_b.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.actor_s_target.parameters(), self.actor_s.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+        for target_param, param in zip(self.actor_b_target.parameters(), self.actor_b.parameters()):
+            target_param.data.copy_(self.tau * param.data + (1.0 - self.tau) * target_param.data)
+            
+        return {
+            'actor_loss_s': actor_loss_s.item(),
+            'actor_loss_b': actor_loss_b.item(),
+            'critic_loss_s': critic_loss_s.item(),
+            'critic_loss_b': critic_loss_b.item()
+        }
+
+    def save(self, path):
+        """
+        Save models to disk.
+        将模型保存至磁盘。
+        """
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.actor_s.state_dict(), os.path.join(path, "actor_s.pth"))
+        torch.save(self.actor_b.state_dict(), os.path.join(path, "actor_b.pth"))
+        torch.save(self.critic_s.state_dict(), os.path.join(path, "critic_s.pth"))
+        torch.save(self.critic_b.state_dict(), os.path.join(path, "critic_b.pth"))
+        
+    def load(self, path):
+        """
+        Load models from disk.
+        从磁盘加载模型。
+        """
+        self.actor_s.load_state_dict(torch.load(os.path.join(path, "actor_s.pth"), map_location=self.device))
+        self.actor_b.load_state_dict(torch.load(os.path.join(path, "actor_b.pth"), map_location=self.device))
+        self.critic_s.load_state_dict(torch.load(os.path.join(path, "critic_s.pth"), map_location=self.device))
+        self.critic_b.load_state_dict(torch.load(os.path.join(path, "critic_b.pth"), map_location=self.device))
+        self.actor_s_target.load_state_dict(self.actor_s.state_dict())
+        self.actor_b_target.load_state_dict(self.actor_b.state_dict())
+        self.critic_s_target.load_state_dict(self.critic_s.state_dict())
+        self.critic_b_target.load_state_dict(self.critic_b.state_dict())

code/baselines/semantic_only.py

@@ -0,0 +1,238 @@
+import os
+import random
+from collections import deque
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from agents.actor import Actor
+from agents.noise import OUNoise
+
+"""
+Baseline: SemanticOnly (仅语义基线)
+=====================================
+Purpose (ablation): 
+- This baseline removes the heterogeneous treatment of different user groups.
+- It treats all users as semantic users and uses a single DDPG policy to control both groups.
+- It serves as an ablation study to demonstrate the benefit of having heterogeneous, specialized policies for semantic vs. traditional users.
+- 目的（消融实验）：该基线移除了对不同用户组的异构处理。它将所有用户视为语义用户，并使用单一的 DDPG 策略同时控制两个用户组。作为消融实验，用于证明为语义用户和传统用户分别设计专门的异构策略的收益。
+
+Difference from Co-MADDPG:
+1. Heterogeneity: Homogeneous policy (all semantic) vs Heterogeneous policies.
+2. Architecture: Single DDPG agent for both groups vs Multi-agent (Co-MADDPG).
+3. 与 Co-MADDPG 的区别：
+   - 异构性：同构策略（全部视为语义用户） vs 异构策略。
+   - 架构：单 DDPG 智能体控制两组 vs 多智能体 (Co-MADDPG)。
+
+Contribution:
+- Contributes to performance analysis regarding user heterogeneity and specialized resource allocation.
+- 贡献：用于关于用户异构性和专门化资源分配的性能分析。
+"""
+
+class SemanticCritic(nn.Module):
+    """
+    Single-agent critic: observation + action → Q-value.
+    单智能体 Critic：观察 + 动作 → Q 值。
+    """
+    def __init__(self, obs_dim, act_dim, hidden_sizes=[256, 256, 128]):
+        super().__init__()
+        assert len(hidden_sizes) == 3
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim + act_dim, hidden_sizes[0]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[0], hidden_sizes[1]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[1], hidden_sizes[2]),
+            nn.ReLU(),
+            nn.Linear(hidden_sizes[2], 1),
+        )
+
+    def forward(self, obs, act):
+        # Forward pass for single agent
+        # 单智能体前向传播
+        return self.net(torch.cat([obs, act], dim=1))
+
+
+class SemanticBuffer:
+    """
+    Replay buffer for SemanticOnly baseline.
+    仅语义基线的重放池。
+    
+    Wrapper that accepts the 9-arg multi-agent push but stores single-agent transitions.
+    接收多智能体 9 参数 push 请求，但内部存储单智能体转换数据。
+    """
+    def __init__(self, capacity):
+        self.buffer = deque(maxlen=capacity)
+
+    def push(self, obs_s, obs_b, act_s, act_b, rew_s, rew_b,
+             next_obs_s, next_obs_b, done=False):
+        """
+        Store only semantic agent's observation/action and average reward.
+        仅存储语义智能体的观察/动作以及平均奖励。
+        """
+        self.buffer.append((
+            np.asarray(obs_s, dtype=np.float32),
+            np.asarray(act_s, dtype=np.float32),
+            float(0.5 * (rew_s + rew_b)),
+            np.asarray(next_obs_s, dtype=np.float32),
+            float(done),
+        ))
+
+    def sample(self, batch_size):
+        """Sample batch."""
+        batch = random.sample(self.buffer, batch_size)
+        obs, act, rew, next_obs, dones = zip(*batch)
+        return (np.array(obs), np.array(act), np.array(rew, dtype=np.float32),
+                np.array(next_obs), np.array(dones, dtype=np.float32))
+
+    def __len__(self):
+        return len(self.buffer)
+
+
+class SemanticOnly:
+    """
+    SemanticOnly algorithm implementation.
+    仅语义算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # Hyperparameters
+        # 超参数
+        self.gamma = config['training']['gamma']
+        self.tau = config['training']['tau']
+        self.batch_size = config['training']['batch_size']
+
+        # Dimensions
+        # 维度
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.act_dim = 3
+
+        # Network configurations
+        # 网络配置
+        hidden_a = config['network']['actor_hidden']
+        critic_hidden = [256, 256, 128] 
+
+        # Single Actor and Critic policy
+        # 单一 Actor 与 Critic 策略
+        self.actor = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_target = Actor(self.obs_dim, self.act_dim, hidden_a).to(self.device)
+        self.actor_target.load_state_dict(self.actor.state_dict())
+
+        self.critic = SemanticCritic(self.obs_dim, self.act_dim, critic_hidden).to(self.device)
+        self.critic_target = SemanticCritic(self.obs_dim, self.act_dim, critic_hidden).to(self.device)
+        self.critic_target.load_state_dict(self.critic.state_dict())
+
+        # Optimizers
+        # 优化器
+        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=config['training']['actor_lr'])
+        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=config['training']['critic_lr'])
+
+        # Buffer and Noise
+        # 重放池与噪声
+        self.replay_buffer = SemanticBuffer(config['training']['buffer_capacity'])
+        self.noise_s = OUNoise(self.act_dim, theta=config['training']['ou_theta'],
+                               sigma_init=config['training']['ou_sigma_init'],
+                               sigma_min=config['training']['ou_sigma_min'])
+        # Alias for compatibility with training loop
+        # 与训练循环兼容的别名
+        self.noise_b = self.noise_s
+
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select actions for both groups using the same policy.
+        使用相同策略为两组用户选择动作。
+        """
+        self.actor.eval()
+        with torch.no_grad():
+            obs_t = torch.FloatTensor(obs_s).unsqueeze(0).to(self.device)
+            act = self.actor(obs_t).cpu().numpy()[0]
+        self.actor.train()
+
+        if explore:
+            # Apply OU noise
+            # 应用 OU 噪声
+            act = np.clip(act + self.noise_s.sample(), 0.0, 1.0)
+        else:
+            act = np.clip(act, 0.0, 1.0)
+
+        # Return the same action for both groups
+        # 为两组用户返回相同的动作
+        return act.copy(), act.copy()
+
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute rewards assuming full cooperation (λ=1).
+        假设完全协作 (λ=1) 计算奖励。
+        
+        Formula: r = 0.5 * (qoe_s + qoe_b)
+        公式说明：由于全部视为语义用户，目标是最大化整体 QoE。
+        """
+        lam = 1.0 
+        r = 0.5 * (qoe_s + qoe_b)
+        return r, r, lam
+
+    def update(self):
+        """
+        Update the single DDPG agent.
+        更新单个 DDPG 智能体。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+
+        # Sample from buffer
+        # 从重放池采样
+        obs, act, rew, next_obs, dones = self.replay_buffer.sample(self.batch_size)
+
+        # To tensors
+        # 转换为张量
+        obs_t = torch.FloatTensor(obs).to(self.device)
+        act_t = torch.FloatTensor(act).to(self.device)
+        rew_t = torch.FloatTensor(rew).unsqueeze(1).to(self.device)
+        next_obs_t = torch.FloatTensor(next_obs).to(self.device)
+        dones_t = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+
+        # 1. Critic update (1. Critic 更新)
+        with torch.no_grad():
+            next_act = self.actor_target(next_obs_t)
+            target_q = rew_t + self.gamma * (1 - dones_t) * self.critic_target(next_obs_t, next_act)
+        
+        current_q = self.critic(obs_t, act_t)
+        critic_loss = F.mse_loss(current_q, target_q)
+        self.critic_optimizer.zero_grad()
+        critic_loss.backward()
+        self.critic_optimizer.step()
+
+        # 2. Actor update (2. Actor 更新)
+        new_act = self.actor(obs_t)
+        actor_loss = -self.critic(obs_t, new_act).mean()
+        self.actor_optimizer.zero_grad()
+        actor_loss.backward()
+        self.actor_optimizer.step()
+
+        # 3. Soft update targets (3. 目标网络软更新)
+        for target, source in [
+            (self.critic_target, self.critic),
+            (self.actor_target, self.actor),
+        ]:
+            for tp, sp in zip(target.parameters(), source.parameters()):
+                tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {'actor_loss': actor_loss.item(), 'critic_loss': critic_loss.item()}
+
+    def save(self, path):
+        """Save models."""
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.actor.state_dict(), os.path.join(path, "actor.pth"))
+        torch.save(self.critic.state_dict(), os.path.join(path, "critic.pth"))
+
+    def load(self, path):
+        """Load models."""
+        self.actor.load_state_dict(torch.load(os.path.join(path, "actor.pth"), map_location=self.device))
+        self.critic.load_state_dict(torch.load(os.path.join(path, "critic.pth"), map_location=self.device))
+        self.actor_target.load_state_dict(self.actor.state_dict())
+        self.critic_target.load_state_dict(self.critic.state_dict())

code/baselines/single_dqn.py

@@ -0,0 +1,296 @@
+import os
+import random
+from collections import deque
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+"""
+Baseline: SingleAgentDQN (单智能体 DQN 基线)
+=====================================
+Purpose (non-MARL baseline): 
+- This baseline represents a traditional single-agent approach to the resource allocation problem.
+- It uses a centralized DQN that controls both groups by discretizing the continuous action space.
+- 目的（非多智能体基线）：该基线代表了解决资源分配问题的传统单智能体方法。它使用中心化 DQN，通过对连续动作空间进行离散化，同时控制两个用户组。
+
+Difference from Co-MADDPG:
+1. Algorithm Class: Non-MARL (DQN) vs MARL (Co-MADDPG).
+2. Action Space: Discrete (48 actions) vs Continuous.
+3. Architecture: Centralized control vs Decentralized execution with CTDE.
+4. Exploration: Epsilon-greedy vs OU Noise.
+5. 与 Co-MADDPG 的区别：
+   - 算法类别：非多智能体 (DQN) vs 多智能体 (Co-MADDPG)。
+   - 动作空间：离散（48 种动作组合） vs 连续。
+   - 架构：中心化控制 vs CTDE 架构下的分布式执行。
+   - 探索机制：ε-greedy vs OU 噪声。
+
+Contribution:
+- Contributes to performance tables showing the limitations of discretization and centralized control in complex multi-user scenarios.
+- 贡献：用于性能表，展示在复杂多用户场景下，动作离散化和中心化控制的局限性。
+"""
+
+# ---- Discrete action mapping (离散动作映射) ----
+# 4 levels for subcarrier fraction, 4 for power fraction, 3 for m_param
+# 子载波比例 4 级，功率比例 4 级，m 参数 3 级
+N_SUB_LEVELS = [0.25, 0.5, 0.75, 1.0]
+P_FRAC_LEVELS = [0.25, 0.5, 0.75, 1.0]
+M_PARAM_LEVELS = [0.33, 0.66, 1.0]
+NUM_ACTIONS = len(N_SUB_LEVELS) * len(P_FRAC_LEVELS) * len(M_PARAM_LEVELS)  # 48 combinations
+
+# Build lookup table: index -> (n_sub_frac, p_frac, m_param)
+# 构建查找表：索引 -> (子载波比例, 功率比例, m 参数)
+_ACTION_TABLE = []
+for n in N_SUB_LEVELS:
+    for p in P_FRAC_LEVELS:
+        for m in M_PARAM_LEVELS:
+            _ACTION_TABLE.append(np.array([n, p, m], dtype=np.float32))
+
+
+class DQNNet(nn.Module):
+    """
+    Simple Fully Connected Q-network.
+    简单的全连接 Q 网络。
+    """
+    def __init__(self, state_dim, num_actions):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(state_dim, 256),
+            nn.ReLU(),
+            nn.Linear(256, 256),
+            nn.ReLU(),
+            nn.Linear(256, num_actions),
+        )
+
+    def forward(self, x):
+        """Map state to Q-values for each discrete action."""
+        return self.net(x)
+
+
+class DQNReplayBuffer:
+    """
+    Wrapper buffer for SingleAgentDQN.
+    单智能体 DQN 的封装重放池。
+    
+    Accepts the multi-agent 9-argument signature but stores transitions suitable for DQN.
+    接收多智能体的 9 参数签名，但内部存储适合 DQN 的转换数据。
+    """
+    def __init__(self, capacity):
+        self.buffer = deque(maxlen=capacity)
+        self._last_action_s_idx = 0
+        self._last_action_b_idx = 0
+
+    def set_last_actions(self, idx_s, idx_b):
+        """Store the discrete action indices used."""
+        self._last_action_s_idx = idx_s
+        self._last_action_b_idx = idx_b
+
+    def push(self, obs_s, obs_b, act_s, act_b, rew_s, rew_b,
+             next_obs_s, next_obs_b, done=False):
+        """
+        Store multi-agent step as a single-agent transition.
+        将多智能体步骤作为单智能体转换存储。
+        """
+        # Concatenate observations for centralized state
+        # 拼接观察值以形成中心化状态
+        state = np.concatenate([np.asarray(obs_s, dtype=np.float32),
+                                np.asarray(obs_b, dtype=np.float32)])
+        next_state = np.concatenate([np.asarray(next_obs_s, dtype=np.float32),
+                                     np.asarray(next_obs_b, dtype=np.float32)])
+        # Average rewards for single-agent scalar reward
+        # 对奖励求平均以获得单智能体标量奖励
+        reward = 0.5 * (float(rew_s) + float(rew_b))
+        self.buffer.append((state, self._last_action_s_idx, self._last_action_b_idx,
+                            reward, next_state, float(done)))
+
+    def sample(self, batch_size):
+        """Sample a batch of transitions."""
+        batch = random.sample(self.buffer, batch_size)
+        states, a_s, a_b, rewards, next_states, dones = zip(*batch)
+        return (np.array(states), np.array(a_s), np.array(a_b),
+                np.array(rewards, dtype=np.float32),
+                np.array(next_states), np.array(dones, dtype=np.float32))
+
+    def __len__(self):
+        return len(self.buffer)
+
+
+class SingleAgentDQN:
+    """
+    SingleAgentDQN algorithm implementation.
+    单智能体 DQN 算法实现。
+    """
+    def __init__(self, config):
+        # Initialize configuration and device
+        # 初始化配置和设备
+        self.config = config
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # Hyperparameters
+        # 超参数
+        self.gamma = config['training']['gamma']
+        self.batch_size = config['training']['batch_size']
+        self.tau = config['training']['tau']
+
+        # Dimensions: Concentated state
+        # 维度：拼接后的状态
+        self.obs_dim = config['env']['num_subcarriers'] + 4
+        self.state_dim = self.obs_dim * 2 
+        self.num_actions = NUM_ACTIONS
+
+        # Two DQN heads: one for semantic (s) actions, one for traditional (b) actions
+        # 两个 DQN 头：一个用于语义动作 (s)，一个用于传统动作 (b)
+        self.q_net_s = DQNNet(self.state_dim, self.num_actions).to(self.device)
+        self.q_net_b = DQNNet(self.state_dim, self.num_actions).to(self.device)
+        self.q_target_s = DQNNet(self.state_dim, self.num_actions).to(self.device)
+        self.q_target_b = DQNNet(self.state_dim, self.num_actions).to(self.device)
+        self.q_target_s.load_state_dict(self.q_net_s.state_dict())
+        self.q_target_b.load_state_dict(self.q_net_b.state_dict())
+
+        # Optimizers
+        # 优化器
+        lr = config['training'].get('actor_lr', 1e-4)
+        self.optimizer_s = torch.optim.Adam(self.q_net_s.parameters(), lr=lr)
+        self.optimizer_b = torch.optim.Adam(self.q_net_b.parameters(), lr=lr)
+
+        # Epsilon-greedy exploration parameters
+        # ε-greedy 探索参数
+        self.epsilon = 1.0
+        self.epsilon_min = 0.01
+        self.epsilon_decay_episodes = 3000
+
+        # Specialized Replay Buffer
+        # 专用的重放池
+        self.replay_buffer = DQNReplayBuffer(config['training']['buffer_capacity'])
+
+        # Discrete action index tracking
+        # 离散动作索引追踪
+        self._last_action_s_idx = 0
+        self._last_action_b_idx = 0
+
+        # EpsilonAdapter: Hack to allow epsilon decay via train.py's existing loop
+        # EpsilonAdapter：用于通过 train.py 现有循环触发 ε 衰减的技巧
+        self.noise_s = type('EpsilonAdapter', (), {
+            'decay_sigma': lambda _, ep: self._decay_epsilon(ep)
+        })()
+
+    def select_action(self, obs_s, obs_b, explore=True):
+        """
+        Select discrete actions using epsilon-greedy policy.
+        使用 ε-greedy 策略选择离散动作。
+        """
+        state = np.concatenate([obs_s, obs_b]).astype(np.float32)
+        state_t = torch.FloatTensor(state).unsqueeze(0).to(self.device)
+
+        if explore and random.random() < self.epsilon:
+            # Random exploration
+            # 随机探索
+            idx_s = random.randrange(self.num_actions)
+            idx_b = random.randrange(self.num_actions)
+        else:
+            # Exploit learned Q-values
+            # 利用已学习的 Q 值
+            self.q_net_s.eval()
+            self.q_net_b.eval()
+            with torch.no_grad():
+                q_s = self.q_net_s(state_t)
+                q_b = self.q_net_b(state_t)
+            self.q_net_s.train()
+            self.q_net_b.train()
+            idx_s = q_s.argmax(dim=1).item()
+            idx_b = q_b.argmax(dim=1).item()
+
+        # Update last indices for the buffer push
+        # 更新用于存入重放池的最后索引
+        self._last_action_s_idx = idx_s
+        self._last_action_b_idx = idx_b
+        self.replay_buffer.set_last_actions(idx_s, idx_b)
+
+        # Return continuous actions from lookup table
+        # 从查找表中返回对应的连续动作
+        return _ACTION_TABLE[idx_s].copy(), _ACTION_TABLE[idx_b].copy()
+
+    def compute_rewards(self, qoe_s, qoe_b, qoe_sys):
+        """
+        Compute scalar reward for single agent.
+        为单智能体计算标量奖励。
+        
+        Formula: r = 0.5 * (qoe_s + qoe_b)
+        公式说明：由于是单智能体控制全局，奖励取两组用户 QoE 的均值。
+        """
+        lam = 0.5
+        r = 0.5 * (qoe_s + qoe_b)
+        return r, r, lam
+
+    def update(self):
+        """
+        Update the Q-networks.
+        更新 Q 网络。
+        """
+        if len(self.replay_buffer) < self.batch_size:
+            return None
+
+        # Sample batch
+        # 采样批量数据
+        states, a_s, a_b, rewards, next_states, dones = \
+            self.replay_buffer.sample(self.batch_size)
+
+        # To tensors
+        # 转换为张量
+        states_t = torch.FloatTensor(states).to(self.device)
+        next_states_t = torch.FloatTensor(next_states).to(self.device)
+        rewards_t = torch.FloatTensor(rewards).unsqueeze(1).to(self.device)
+        dones_t = torch.FloatTensor(dones).unsqueeze(1).to(self.device)
+        a_s_t = torch.LongTensor(a_s).unsqueeze(1).to(self.device)
+        a_b_t = torch.LongTensor(a_b).unsqueeze(1).to(self.device)
+
+        # 1. Update Semantic Head (1. 更新语义分支)
+        q_values_s = self.q_net_s(states_t).gather(1, a_s_t)
+        with torch.no_grad():
+            next_q_s = self.q_target_s(next_states_t).max(1, keepdim=True)[0]
+            target_s = rewards_t + self.gamma * (1 - dones_t) * next_q_s
+        loss_s = F.mse_loss(q_values_s, target_s)
+        self.optimizer_s.zero_grad()
+        loss_s.backward()
+        self.optimizer_s.step()
+
+        # 2. Update Traditional Head (2. 更新传统分支)
+        q_values_b = self.q_net_b(states_t).gather(1, a_b_t)
+        with torch.no_grad():
+            next_q_b = self.q_target_b(next_states_t).max(1, keepdim=True)[0]
+            target_b = rewards_t + self.gamma * (1 - dones_t) * next_q_b
+        loss_b = F.mse_loss(q_values_b, target_b)
+        self.optimizer_b.zero_grad()
+        loss_b.backward()
+        self.optimizer_b.step()
+
+        # 3. Soft update target networks (3. 目标网络软更新)
+        for target, source in [
+            (self.q_target_s, self.q_net_s),
+            (self.q_target_b, self.q_net_b),
+        ]:
+            for tp, sp in zip(target.parameters(), source.parameters()):
+                tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {'loss_s': loss_s.item(), 'loss_b': loss_b.item()}
+
+    def _decay_epsilon(self, episode):
+        """
+        Decay epsilon over episodes.
+        随训练轮数衰减 ε。
+        """
+        frac = min(1.0, episode / max(1, self.epsilon_decay_episodes))
+        self.epsilon = self.epsilon + frac * (self.epsilon_min - self.epsilon)
+
+    def save(self, path):
+        """Save Q-nets."""
+        os.makedirs(path, exist_ok=True)
+        torch.save(self.q_net_s.state_dict(), os.path.join(path, "q_net_s.pth"))
+        torch.save(self.q_net_b.state_dict(), os.path.join(path, "q_net_b.pth"))
+
+    def load(self, path):
+        """Load Q-nets."""
+        self.q_net_s.load_state_dict(torch.load(os.path.join(path, "q_net_s.pth"), map_location=self.device))
+        self.q_net_b.load_state_dict(torch.load(os.path.join(path, "q_net_b.pth"), map_location=self.device))
+        self.q_target_s.load_state_dict(self.q_net_s.state_dict())
+        self.q_target_b.load_state_dict(self.q_net_b.state_dict())