1{
2  "error": "Failed to parse tool arguments: Expecting ',' delimiter: line 1 column 109 (char 108)"
3}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_value_discrete.py",
4  "start_line": 174,
5  "end_line": 242,
6  "content": "class QNetwork(nn.Module):\n    \"\"\"Q-network: MLPEncoder (fixed) + head. Output: Q-values per action.\"\"\"\n\n    def __init__(self, obs_dim, n_actions):\n        super().__init__()\n        self.encoder = MLPEncoder(obs_dim)\n        self.head = nn.Linear(ENCODER_FEATURE_DIM, n_actions)\n\n    def forward(self, obs):\n        features = self.encoder(obs)\n        return self.head(features)\n\n\nclass ValueAlgorithm:\n    \"\"\"Munchausen Double DQN: entropy-augmented bootstrapping with double\n    Q-learning. Adds a scaled log-policy bonus to rewards and uses soft\n    value targets, providing implicit KL regularization. Uses the online\n    network's policy for next-state action ranking (double Q-learning)\n    while evaluating with the target network's Q-values.\n    \"\"\"\n\n    def __init__(self, obs_dim, n_actions, device, args):\n        self.device = device\n        self.n_actions = n_actions\n        self.gamma = args.gamma\n        self.total_it = 0\n        self.m_alpha = 0.9   # Munchausen scaling factor\n        self.m_tau = 0.03    # entropy temperature\n        self.lo = -1.0       # log-policy clipping lower bound\n        self.q_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network.load_state_dict(self.q_network.state_dict())\n        self.optimizer = optim.Adam(self.q_network.parameters(), lr=args.learning_rate)\n\n    def select_action(self, obs, epsilon):\n        if random.random() < epsilon:\n            return random.randint(0, self.n_actions - 1)\n        obs_t = torch.tensor(obs.reshape(1, -1), device=self.device, dtype=torch.float32)\n        q_values = self.q_network(obs_t)\n        return torch.argmax(q_values, dim=1).item()\n\n    def update(self, batch, global_step):\n        self.total_it += 1\n        obs, next_obs, actions, rewards, dones = batch\n        with torch.no_grad():\n            # Double Q: use online net's policy, target net's Q-values\n            q_next_online = self.q_network(next_obs)\n            q_next_target = self.target_network(next_obs)\n            # Soft policy from online network for next state\n            log_pi_next = F.log_softmax(q_next_online / self.m_tau, dim=1)\n            pi_next = log_pi_next.exp()\n            # Soft value using target Q-values but online policy (Double)\n            v_next = (pi_next * (q_next_target - self.m_tau * log_pi_next)).sum(1)\n            # Munchausen bonus: log-policy for current action (target net)\n            q_curr_tgt = self.target_network(obs)\n            lp = F.log_softmax(q_curr_tgt / self.m_tau, dim=1)\n            lp_a = lp[torch.arange(len(obs)), actions].clamp(self.lo, 0)\n            # Augmented Bellman target\n            td_target = rewards + self.m_alpha * self.m_tau * lp_a + \\\n                        self.gamma * (1 - dones) * v_next\n        old_val = self.q_network(obs).gather(1, actions.unsqueeze(1)).squeeze(1)\n        loss = F.smooth_l1_loss(old_val, td_target)\n        self.optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(self.q_network.parameters(), max_norm=10.0)\n        self.optimizer.step()\n        return {\"td_loss\": loss.item(), \"q_values\": old_val.mean().item()}"
7}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_value_discrete.py",
4  "start_line": 174,
5  "end_line": 240,
6  "content": "class QNetwork(nn.Module):\n    \"\"\"Q-network: MLPEncoder (fixed) + head. Output: Q-values per action.\"\"\"\n\n    def __init__(self, obs_dim, n_actions):\n        super().__init__()\n        self.encoder = MLPEncoder(obs_dim)\n        self.head = nn.Linear(ENCODER_FEATURE_DIM, n_actions)\n\n    def forward(self, obs):\n        features = self.encoder(obs)\n        return self.head(features)\n\n\nclass ValueAlgorithm:\n    \"\"\"Pessimistic Munchausen Double DQN (PM-DDQN).\n    Novel combination of: (1) Double Q-learning for action selection,\n    (2) Munchausen reward augmentation adding scaled log-policy bonus for\n    implicit KL regularization, and (3) asymmetric (expectile) Huber loss\n    that penalizes Q-value overestimation more than underestimation,\n    pushing learned values toward a pessimistic expectile of the target.\n    \"\"\"\n\n    def __init__(self, obs_dim, n_actions, device, args):\n        self.device = device\n        self.n_actions = n_actions\n        self.gamma = args.gamma\n        self.total_it = 0\n        self.m_alpha = 0.9    # Munchausen scaling\n        self.m_tau = 0.03     # entropy temperature for log-policy\n        self.lo = -1.0        # log-policy clipping bound\n        self.asym_tau = 0.6   # expectile: >0.5 penalizes overestimation more\n        self.q_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network.load_state_dict(self.q_network.state_dict())\n        self.optimizer = optim.Adam(self.q_network.parameters(), lr=args.learning_rate)\n\n    def select_action(self, obs, epsilon):\n        if random.random() < epsilon:\n            return random.randint(0, self.n_actions - 1)\n        obs_t = torch.tensor(obs.reshape(1, -1), device=self.device, dtype=torch.float32)\n        q_values = self.q_network(obs_t)\n        return torch.argmax(q_values, dim=1).item()\n\n    def update(self, batch, global_step):\n        self.total_it += 1\n        obs, next_obs, actions, rewards, dones = batch\n        with torch.no_grad():\n            # Double DQN: online net selects action, target net evaluates\n            next_act = self.q_network(next_obs).argmax(dim=1)\n            tgt_q = self.target_network(next_obs)\n            next_q = tgt_q[torch.arange(len(next_obs)), next_act]\n            # Munchausen reward augmentation via target net log-policy\n            q_s = self.target_network(obs)\n            lp = F.log_softmax(q_s / self.m_tau, dim=1)\n            lp_a = lp[torch.arange(len(obs)), actions].clamp(self.lo, 0)\n            aug_r = rewards + self.m_alpha * self.m_tau * lp_a\n            td_target = aug_r + self.gamma * (1 - dones) * next_q\n        old_val = self.q_network(obs).gather(1, actions.unsqueeze(1)).squeeze(1)\n        # Asymmetric Huber loss (expectile): penalizes overestimation more\n        delta = old_val - td_target\n        abs_d = torch.abs(delta)\n        huber = torch.where(abs_d <= 1.0, 0.5 * delta ** 2, abs_d - 0.5)\n        w = torch.where(delta.detach() > 0, self.asym_tau, 1.0 - self.asym_tau)\n        loss = (w * huber).mean()\n        self.optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(self.q_network.parameters(), max_norm=10.0)\n        self.optimizer.step()\n        return {\"td_loss\": loss.item(), \"q_values\": old_val.mean().item()}"
7}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_value_discrete.py",
4  "start_line": 174,
5  "end_line": 242,
6  "content": "class QNetwork(nn.Module):\n    \"\"\"Q-network: MLPEncoder (fixed) + head. Output: Q-values per action.\"\"\"\n\n    def __init__(self, obs_dim, n_actions):\n        super().__init__()\n        self.encoder = MLPEncoder(obs_dim)\n        self.head = nn.Linear(ENCODER_FEATURE_DIM, n_actions)\n\n    def forward(self, obs):\n        features = self.encoder(obs)\n        return self.head(features)\n\n\nclass ValueAlgorithm:\n    \"\"\"Ensembled Munchausen DQN (EM-DQN). Novel combination of:\n    (1) Munchausen reward augmentation: adds scaled log-policy bonus from\n    the target network, providing implicit KL regularization that penalizes\n    actions inconsistent with the current policy.\n    (2) Ensemble action selection: averages online and target Q-networks\n    for training data collection, yielding more robust exploration by\n    reducing the impact of any single network's overestimation.\n    (3) MSE loss for proper gradient scaling on large TD errors.\n    \"\"\"\n\n    def __init__(self, obs_dim, n_actions, device, args):\n        self.device = device\n        self.n_actions = n_actions\n        self.gamma = args.gamma\n        self.total_it = 0\n        self.m_alpha = 0.9    # Munchausen scaling factor\n        self.m_tau = 0.03     # entropy temperature for log-policy\n        self.lo = -1.0        # log-policy clipping lower bound\n        self.q_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network = QNetwork(obs_dim, n_actions).to(device)\n        self.target_network.load_state_dict(self.q_network.state_dict())\n        self.optimizer = optim.Adam(self.q_network.parameters(), lr=args.learning_rate)\n\n    def select_action(self, obs, epsilon):\n        \"\"\"Ensemble epsilon-greedy: average online+target Q for selection.\"\"\"\n        if random.random() < epsilon:\n            return random.randint(0, self.n_actions - 1)\n        obs_t = torch.tensor(obs.reshape(1, -1), device=self.device, dtype=torch.float32)\n        with torch.no_grad():\n            q_on = self.q_network(obs_t)\n            q_tg = self.target_network(obs_t)\n            q_ens = 0.5 * q_on + 0.5 * q_tg\n        return torch.argmax(q_ens, dim=1).item()\n\n    def update(self, batch, global_step):\n        self.total_it += 1\n        obs, next_obs, actions, rewards, dones = batch\n        with torch.no_grad():\n            # Standard max target from target network (proven stable)\n            target_max = self.target_network(next_obs).max(dim=1)[0]\n            # Munchausen reward augmentation via target net's log-policy\n            q_s = self.target_network(obs)\n            lp = F.log_softmax(q_s / self.m_tau, dim=1)\n            lp_a = lp[torch.arange(len(obs)), actions].clamp(self.lo, 0)\n            aug_r = rewards + self.m_alpha * self.m_tau * lp_a\n            td_target = aug_r + self.gamma * (1 - dones) * target_max\n        old_val = self.q_network(obs).gather(1, actions.unsqueeze(1)).squeeze(1)\n        loss = F.mse_loss(old_val, td_target)\n        self.optimizer.zero_grad()\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(self.q_network.parameters(), max_norm=10.0)\n        self.optimizer.step()\n        return {\"td_loss\": loss.item(), \"q_values\": old_val.mean().item()}"
7}