1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_offpolicy_continuous.py",
4  "start_line": 153,
5  "end_line": 244,
6  "content": "class Actor(nn.Module):\n    \"\"\"Deterministic actor with tanh squashing.\"\"\"\n\n    def __init__(self, obs_dim, action_dim, max_action):\n        super().__init__()\n        self.max_action = max_action\n        self.fc1 = nn.Linear(obs_dim, 256)\n        self.fc2 = nn.Linear(256, 256)\n        self.fc_mu = nn.Linear(256, action_dim)\n        self.register_buffer(\"action_scale\", torch.tensor(max_action, dtype=torch.float32))\n\n    def forward(self, obs):\n        x = F.relu(self.fc1(obs))\n        x = F.relu(self.fc2(x))\n        return torch.tanh(self.fc_mu(x)) * self.action_scale\n\n    @torch.no_grad()\n    def get_action(self, obs):\n        return self.forward(obs)\n\n\nclass QNetwork(nn.Module):\n    \"\"\"Q-function Q(s, a) -> scalar.\"\"\"\n\n    def __init__(self, obs_dim, action_dim):\n        super().__init__()\n        self.fc1 = nn.Linear(obs_dim + action_dim, 256)\n        self.fc2 = nn.Linear(256, 256)\n        self.fc3 = nn.Linear(256, 1)\n\n    def forward(self, obs, action):\n        x = torch.cat([obs, action], dim=-1)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        return self.fc3(x)\n\n\nclass OffPolicyAlgorithm:\n    \"\"\"APAC: Adaptive Pessimistic Actor-Critic.\n\n    Novel contributions over TD3:\n    1. Per-sample adaptive pessimism: target Q uses a disagreement-dependent\n       blend of min/mean critics. Samples where critics disagree more get\n       more conservative targets (uncertainty-aware value estimation).\n    2. Dual-critic actor gradient: actor maximizes mean(Q1,Q2) for\n       lower-variance policy gradients using all critic information.\n    \"\"\"\n\n    def __init__(self, obs_dim, action_dim, max_action, device, args):\n        self.device = device\n        self.max_action = max_action\n        self.gamma = args.gamma\n        self.tau = args.tau\n        self.exploration_noise = args.exploration_noise\n        self.policy_frequency = args.policy_frequency\n        self.total_it = 0\n        self.policy_noise = 0.2\n        self.noise_clip = 0.5\n        # Adaptive pessimism: base_pess when disagreement == EMA,\n        # increases toward 1.0 for high-disagreement samples,\n        # decreases to 0.5 for low-disagreement samples.\n        self.base_pessimism = 0.75\n        self.pessimism_sensitivity = 0.25\n        self.disagreement_ema = -1.0  # sentinel for uninitialized\n        self.ema_decay = 0.999\n\n        self.actor = Actor(obs_dim, action_dim, max_action).to(device)\n        self.target_actor = Actor(obs_dim, action_dim, max_action).to(device)\n        self.target_actor.load_state_dict(self.actor.state_dict())\n        self.qf1 = QNetwork(obs_dim, action_dim).to(device)\n        self.qf2 = QNetwork(obs_dim, action_dim).to(device)\n        self.qf1_target = QNetwork(obs_dim, action_dim).to(device)\n        self.qf2_target = QNetwork(obs_dim, action_dim).to(device)\n        self.qf1_target.load_state_dict(self.qf1.state_dict())\n        self.qf2_target.load_state_dict(self.qf2.state_dict())\n        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=args.learning_rate)\n        self.q_optimizer = optim.Adam(\n            list(self.qf1.parameters()) + list(self.qf2.parameters()),\n            lr=args.learning_rate,\n        )\n\n    def select_action(self, obs):\n        obs_t = torch.tensor(obs.reshape(1, -1), device=self.device, dtype=torch.float32)\n        with torch.no_grad():\n            action = self.actor(obs_t).cpu().numpy().flatten()\n        noise = np.random.normal(0, self.max_action * self.exploration_noise, size=action.shape)\n        return np.clip(action + noise, -self.max_action, self.max_action)\n\n    def update(self, batch):\n        self.total_it += 1\n        obs, next_obs, actions, rewards, dones = batch\n        with torch.no_grad():\n            noise = (torch.randn_like(actions) * self.policy_noise).clamp(\n                -self.noise_clip, self.noise_clip) * self.max_action\n            next_actions = (self.target_actor(next_obs) + noise).clamp(\n                -self.max_action, self.max_action)\n            tq1 = self.qf1_target(next_obs, next_actions).view(-1)\n            tq2 = self.qf2_target(next_obs, next_actions).view(-1)\n            # Per-sample adaptive pessimism based on critic disagreement\n            disag = torch.abs(tq1 - tq2)\n            bd = disag.mean().item()\n            if self.disagreement_ema < 0:\n                self.disagreement_ema = bd\n            else:\n                self.disagreement_ema = (self.ema_decay * self.disagreement_ema\n                                         + (1 - self.ema_decay) * bd)\n            if self.disagreement_ema > 1e-8:\n                ratio = disag / self.disagreement_ema\n                pess = (self.base_pessimism\n                        + self.pessimism_sensitivity * (ratio - 1.0)).clamp(0.5, 1.0)\n            else:\n                pess = torch.ones_like(tq1)\n            min_q = torch.min(tq1, tq2)\n            mean_q = 0.5 * (tq1 + tq2)\n            target_q = pess * min_q + (1 - pess) * mean_q\n            td_target = rewards + (1 - dones) * self.gamma * target_q\n        q1 = self.qf1(obs, actions).view(-1)\n        q2 = self.qf2(obs, actions).view(-1)\n        critic_loss = F.mse_loss(q1, td_target) + F.mse_loss(q2, td_target)\n        self.q_optimizer.zero_grad()\n        critic_loss.backward()\n        self.q_optimizer.step()\n        actor_loss_val = 0.0\n        if self.total_it % self.policy_frequency == 0:\n            pi = self.actor(obs)\n            q1_pi = self.qf1(obs, pi).view(-1)\n            q2_pi = self.qf2(obs, pi).view(-1)\n            # Dual-critic actor loss: mean of both critics\n            actor_loss = -0.5 * (q1_pi + q2_pi).mean()\n            self.actor_optimizer.zero_grad()\n            actor_loss.backward()\n            self.actor_optimizer.step()\n            actor_loss_val = actor_loss.item()\n            soft_update(self.target_actor, self.actor, self.tau)\n            soft_update(self.qf1_target, self.qf1, self.tau)\n            soft_update(self.qf2_target, self.qf2, self.tau)\n        return {\"critic_loss\": critic_loss.item(), \"actor_loss\": actor_loss_val}\n"
7}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_offpolicy_continuous.py",
4  "start_line": 211,
5  "end_line": 217,
6  "content": "        # Adaptive pessimism: base_pess when disagreement == EMA,\n        # increases toward 1.0 for high-disagreement samples,\n        # decreases to 0.4 for low-disagreement samples.\n        self.base_pessimism = 0.65\n        self.pessimism_sensitivity = 0.35\n        self.disagreement_ema = -1.0  # sentinel for uninitialized\n        self.ema_decay = 0.999"
7}

1{
2  "submit": 1
3}