1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_onpolicy_continuous.py",
4  "start_line": 175,
5  "end_line": 221,
6  "content": "    def get_action_and_value(self, obs, action=None):\n        action_mean = self.actor_mean(obs)\n        action_logstd = self.actor_logstd.expand_as(action_mean)\n        action_std = torch.exp(action_logstd)\n\n        # Random Perturbation Optimization (RPO): inject uniform noise into\n        # the action mean before constructing the Gaussian. Each forward pass\n        # sees a different perturbation, which randomizes the importance-\n        # sampling ratio and implicitly enlarges the exploration region.\n        _rpo_alpha = 0.5\n        z = torch.FloatTensor(action_mean.shape).uniform_(\n            -_rpo_alpha, _rpo_alpha\n        ).to(action_mean.device)\n        action_mean = action_mean + z\n\n        probs = Normal(action_mean, action_std)\n        if action is None:\n            action = probs.sample()\n        return action, probs.log_prob(action).sum(1), probs.entropy().sum(1), self.critic(obs)\n\n\ndef compute_losses(agent, mb_obs, mb_actions, mb_logprobs, mb_advantages, mb_returns, mb_values, args):\n    \"\"\"Stochastic Trust-Region PPO (STP-PPO).\n\n    Combines RPO exploration (via perturbed action means) with:\n    1. PPO clipped surrogate objective\n    2. Clipped value-function loss\n    3. Advantage-calibrated entropy: entropy bonus is modulated per sample\n       so that states with negative advantages (under-performing) receive a\n       larger entropy push (explore more) while states with positive\n       advantages (well-performing) get a smaller push (exploit more).\n    \"\"\"\n    # Hardcoded hyperparameters for our novel components\n    _ent_coef = 0.005        # base entropy coefficient (overrides default 0.0)\n    _adv_ent_modulation = 0.4  # how much advantages modulate entropy weight\n\n    _, newlogprob, entropy, newvalue = agent.get_action_and_value(mb_obs, mb_actions)\n    logratio = newlogprob - mb_logprobs\n    ratio = logratio.exp()\n\n    with torch.no_grad():\n        approx_kl = ((ratio - 1) - logratio).mean()\n        clipfrac = ((ratio - 1.0).abs() > args.clip_coef).float().mean().item()\n\n    # ---- Policy loss: PPO clipped surrogate ----\n    pg_loss1 = -mb_advantages * ratio\n    pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - args.clip_coef, 1 + args.clip_coef)\n    pg_loss = torch.max(pg_loss1, pg_loss2).mean()\n\n    # ---- Value loss: clipped ----\n    newvalue = newvalue.view(-1)\n    if args.clip_vloss:\n        v_loss_unclipped = (newvalue - mb_returns) ** 2\n        v_clipped = mb_values + torch.clamp(\n            newvalue - mb_values, -args.clip_coef, args.clip_coef\n        )\n        v_loss_clipped = (v_clipped - mb_returns) ** 2\n        v_loss = 0.5 * torch.max(v_loss_unclipped, v_loss_clipped).mean()\n    else:\n        v_loss = 0.5 * ((newvalue - mb_returns) ** 2).mean()\n\n    # ---- Advantage-Calibrated Entropy (ACE) ----\n    # States with negative normalised advantages -> weight > 1 (explore more)\n    # States with positive normalised advantages -> weight < 1 (exploit more)\n    with torch.no_grad():\n        ace_weight = 1.0 - _adv_ent_modulation * torch.tanh(mb_advantages)\n    entropy_loss = (entropy * ace_weight).mean()\n\n    loss = pg_loss - _ent_coef * entropy_loss + v_loss * args.vf_coef\n\n    return loss, pg_loss, v_loss, entropy_loss, approx_kl, clipfrac\n\n\n"
7}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_onpolicy_continuous.py",
4  "start_line": 175,
5  "end_line": 247,
6  "content": "    def get_action_and_value(self, obs, action=None):\n        action_mean = self.actor_mean(obs)\n        action_logstd = self.actor_logstd.expand_as(action_mean)\n        action_std = torch.exp(action_logstd)\n\n        # Scale-Adaptive Random Perturbation (SARP):\n        # Unlike fixed-scale RPO, perturbation scales with the current policy\n        # std so it shrinks as the policy becomes more certain, preventing\n        # over-perturbation in precise-control tasks while preserving\n        # exploration benefits during early training.\n        _sarp_alpha = 0.15\n        z = torch.FloatTensor(action_mean.shape).uniform_(-1.0, 1.0).to(action_mean.device)\n        action_mean = action_mean + _sarp_alpha * action_std.detach() * z\n\n        probs = Normal(action_mean, action_std)\n        if action is None:\n            action = probs.sample()\n        return action, probs.log_prob(action).sum(1), probs.entropy().sum(1), self.critic(obs)\n\n\ndef compute_losses(agent, mb_obs, mb_actions, mb_logprobs, mb_advantages, mb_returns, mb_values, args):\n    \"\"\"Advantage-Weighted PPO with Scale-Adaptive Perturbation (AW-SARP-PPO).\n\n    Novel combination:\n    1. SARP exploration in get_action_and_value (std-proportional perturbation)\n    2. Exponential advantage weighting: samples with higher advantages get\n       larger gradient weight, focusing policy updates on clearly beneficial\n       actions — bridging PPO and AWR philosophies.\n    3. PPO clipped surrogate ensures trust region stability.\n    4. Clipped value function loss.\n    \"\"\"\n    _awr_beta = 2.0         # advantage weighting temperature\n    _awr_max_weight = 10.0   # stability clamp\n\n    _, newlogprob, entropy, newvalue = agent.get_action_and_value(mb_obs, mb_actions)\n    logratio = newlogprob - mb_logprobs\n    ratio = logratio.exp()\n\n    with torch.no_grad():\n        approx_kl = ((ratio - 1) - logratio).mean()\n        clipfrac = ((ratio - 1.0).abs() > args.clip_coef).float().mean().item()\n\n    # ---- Advantage-based importance weights (AWR-style) ----\n    # Exponential weighting focuses gradient on high-advantage samples.\n    # Since weights > 0, the max(pg1, pg2) factorises cleanly:\n    #   max(w*f, w*g) = w * max(f, g)  =>  correct PPO clipping behaviour preserved.\n    with torch.no_grad():\n        weights = torch.exp(mb_advantages / _awr_beta)\n        weights = torch.clamp(weights, max=_awr_max_weight)\n        weights = weights / (weights.mean() + 1e-8)   # normalise to mean 1\n\n    # ---- Policy loss: weighted PPO clipped surrogate ----\n    pg_loss1 = -mb_advantages * ratio * weights\n    pg_loss2 = -mb_advantages * torch.clamp(ratio, 1 - args.clip_coef, 1 + args.clip_coef) * weights\n    pg_loss = torch.max(pg_loss1, pg_loss2).mean()\n\n    # ---- Value loss: clipped ----\n    newvalue = newvalue.view(-1)\n    if args.clip_vloss:\n        v_loss_unclipped = (newvalue - mb_returns) ** 2\n        v_clipped = mb_values + torch.clamp(\n            newvalue - mb_values, -args.clip_coef, args.clip_coef\n        )\n        v_loss_clipped = (v_clipped - mb_returns) ** 2\n        v_loss = 0.5 * torch.max(v_loss_unclipped, v_loss_clipped).mean()\n    else:\n        v_loss = 0.5 * ((newvalue - mb_returns) ** 2).mean()\n\n    entropy_loss = entropy.mean()\n    loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef\n\n    return loss, pg_loss, v_loss, entropy_loss, approx_kl, clipfrac\n\n\n"
7}

1{
2  "op": "replace",
3  "filename": "cleanrl/cleanrl/custom_onpolicy_continuous.py",
4  "start_line": 175,
5  "end_line": 248,
6  "content": "    def get_action_and_value(self, obs, action=None):\n        action_mean = self.actor_mean(obs)\n        action_logstd = self.actor_logstd.expand_as(action_mean)\n        action_std = torch.exp(action_logstd)\n        probs = Normal(action_mean, action_std)\n        if action is None:\n            action = probs.sample()\n        return action, probs.log_prob(action).sum(1), probs.entropy().sum(1), self.critic(obs)\n\n\ndef compute_losses(agent, mb_obs, mb_actions, mb_logprobs, mb_advantages, mb_returns, mb_values, args):\n    \"\"\"Asymmetric-Clip PPO with Unconstrained Value Learning (ACU-PPO).\n\n    Novel contributions over standard PPO:\n    1. Asymmetric policy clipping: positive advantages use a wider upper clip\n       (1 + 2*epsilon) allowing more aggressive exploitation of discovered\n       good actions, while negative advantages retain standard symmetric clip\n       for conservative avoidance of harmful actions.  This addresses PPO's\n       over-conservative treatment of high-advantage transitions without\n       sacrificing trust-region stability for negative-advantage corrections.\n    2. Unconstrained value loss: removes the value-function clipping used in\n       standard PPO. Value clipping can impede value accuracy by preventing\n       necessary corrections; removing it allows faster convergence to\n       accurate value estimates that improve advantage signal quality.\n    \"\"\"\n    _pos_clip_mult = 2.0   # upper clip multiplier for positive advantages\n\n    _, newlogprob, entropy, newvalue = agent.get_action_and_value(mb_obs, mb_actions)\n    logratio = newlogprob - mb_logprobs\n    ratio = logratio.exp()\n\n    with torch.no_grad():\n        approx_kl = ((ratio - 1) - logratio).mean()\n        clipfrac = ((ratio - 1.0).abs() > args.clip_coef).float().mean().item()\n\n    # ---- Asymmetric clipping ----\n    clip_lo = 1.0 - args.clip_coef\n    clip_hi_pos = 1.0 + args.clip_coef * _pos_clip_mult   # wider for A >= 0\n    clip_hi_neg = 1.0 + args.clip_coef                    # standard for A < 0\n\n    ratio_clip_pos = torch.clamp(ratio, clip_lo, clip_hi_pos)\n    ratio_clip_neg = torch.clamp(ratio, clip_lo, clip_hi_neg)\n    ratio_clipped = torch.where(mb_advantages >= 0, ratio_clip_pos, ratio_clip_neg)\n\n    pg_loss1 = -mb_advantages * ratio\n    pg_loss2 = -mb_advantages * ratio_clipped\n    pg_loss = torch.max(pg_loss1, pg_loss2).mean()\n\n    # ---- Unconstrained value loss (plain MSE) ----\n    newvalue = newvalue.view(-1)\n    v_loss = 0.5 * ((newvalue - mb_returns) ** 2).mean()\n\n    entropy_loss = entropy.mean()\n    loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef\n\n    return loss, pg_loss, v_loss, entropy_loss, approx_kl, clipfrac\n\n\n"
7}