1{
2  "op": "replace",
3  "filename": "penalized-bilevel-gradient-descent/mlsbench/custom_strategy.py",
4  "start_line": 216,
5  "end_line": 253,
6  "content": "def get_toy_strategy() -> ToyStrategy:\n    # Novel: reduce gamma from 10→7 to increase effective step size (α=α0/γ)\n    # from 0.01 to ~0.0143, a 43% speedup per step while remaining within\n    # the theoretical convergence bound (2/L where L = L_f + γ·L_g).\n    # This trades slight lower-level precision for significantly faster\n    # convergence to stationarity.\n    return ToyStrategy(\n        method=\"v_pbgd\",\n        gams=(7.0,),\n        alpha0=0.1,\n    )\n\n\ndef get_hyperclean_strategy(net: str) -> HypercleanStrategy:\n    if net == \"linear\":\n        # Novel: \"Enhanced inner tracking\" — increase inner_itr from 1→2\n        # so the inner model better approximates the value function v(x),\n        # producing more informative penalty gradients for cleaning weights.\n        # Also slightly higher gamma_max and longer schedule for stronger\n        # penalty enforcement, plus more outer iterations.\n        return HypercleanStrategy(\n            method=\"v_pbgd\",\n            lrx=0.1,\n            lry=0.1,\n            lr_inner=0.02,\n            gamma_init=0.0,\n            gamma_max=0.3,\n            gamma_argmax_step=30000,\n            outer_itr=50000,\n            inner_itr=2,\n            reg=0.0,\n            eval_interval=10,\n        )\n    if net == \"mlp\":\n        # Use g_pbgd (best baseline for MLP at 92.38%) with a slower\n        # penalty ramp (35k vs 30k steps) for smoother transition,\n        # slightly higher gamma_max, and more outer iterations.\n        return HypercleanStrategy(\n            method=\"g_pbgd\",\n            lrx=0.5,\n            lry=0.5,\n            gamma_init=0.0,\n            gamma_max=40.0,\n            gamma_argmax_step=35000,\n            outer_itr=60000,\n            reg=0.0,\n            eval_interval=10,\n        )\n    raise ValueError(f\"Unsupported network: {net}\")\n"
7}

1{
2  "op": "replace",
3  "filename": "penalized-bilevel-gradient-descent/mlsbench/custom_strategy.py",
4  "start_line": 216,
5  "end_line": 253,
6  "content": "def get_toy_strategy() -> ToyStrategy:\n    # Increase alpha0 from 0.1 to 0.2 for 2x larger step size (alpha=0.02).\n    # Stability analysis: max Hessian eigenvalue ~96, so alpha < 2/96 ≈ 0.021.\n    # alpha=0.02 is just within the stability bound, roughly halving convergence steps.\n    return ToyStrategy(\n        method=\"v_pbgd\",\n        gams=(10.0,),\n        alpha0=0.2,\n    )\n\n\ndef get_hyperclean_strategy(net: str) -> HypercleanStrategy:\n    if net == \"linear\":\n        # V-PBGD with enhanced inner tracking: inner_itr=3 gives 3x better\n        # lower-level value approximation per outer step. Combined with slightly\n        # stronger penalty (gamma_max=0.3) and more outer iterations (50k),\n        # this improves the penalty gradient quality throughout training.\n        return HypercleanStrategy(\n            method=\"v_pbgd\",\n            lrx=0.1,\n            lry=0.1,\n            lr_inner=0.01,\n            gamma_init=0.0,\n            gamma_max=0.3,\n            gamma_argmax_step=25000,\n            outer_itr=50000,\n            inner_itr=3,\n            reg=0.0,\n            eval_interval=10,\n        )\n    if net == \"mlp\":\n        # G-PBGD with extended progressive penalty schedule: slower gamma ramp\n        # (argmax at 35k) with higher gamma_max=42 and more iterations (65k).\n        # The lr_decay = 1/gamma acts as adaptive learning rate reduction,\n        # allowing fine-grained optimization in later stages.\n        return HypercleanStrategy(\n            method=\"g_pbgd\",\n            lrx=0.5,\n            lry=0.5,\n            gamma_init=0.0,\n            gamma_max=42.0,\n            gamma_argmax_step=35000,\n            outer_itr=65000,\n            reg=0.0,\n            eval_interval=10,\n        )\n    raise ValueError(f\"Unsupported network: {net}\")\n"
7}

1{
2  "op": "replace",
3  "filename": "penalized-bilevel-gradient-descent/mlsbench/custom_strategy.py",
4  "start_line": 216,
5  "end_line": 262,
6  "content": "def get_toy_strategy() -> ToyStrategy:\n    # Use gamma=6 (instead of 10) to allow a larger stable step size.\n    # With alpha = 0.18/6 = 0.03, the Lipschitz bound for the penalized\n    # objective at gamma=6 gives L ≈ 52, so alpha < 2/52 ≈ 0.038: safe.\n    # This 3x step-size increase approximately triples convergence speed.\n    return ToyStrategy(\n        method=\"v_pbgd\",\n        gams=(6.0,),\n        alpha0=0.18,\n    )\n\n\ndef get_hyperclean_strategy(net: str) -> HypercleanStrategy:\n    if net == \"linear\":\n        # V-PBGD with stronger penalty signal and slightly faster x learning.\n        # gamma_max=0.3 (vs 0.2) provides ~50% stronger bilevel coupling;\n        # faster gamma ramp (20k vs 30k) reaches full penalty earlier, giving\n        # 28k steps at max penalty vs only 10k in the baseline.\n        # lrx=0.12 allows slightly faster data-weight adaptation.\n        return HypercleanStrategy(\n            method=\"v_pbgd\",\n            lrx=0.12,\n            lry=0.1,\n            lr_inner=0.01,\n            gamma_init=0.0,\n            gamma_max=0.3,\n            gamma_argmax_step=20000,\n            outer_itr=48000,\n            inner_itr=1,\n            reg=0.0,\n            eval_interval=10,\n        )\n    if net == \"mlp\":\n        # G-PBGD with L2 regularization (reg=0.001) for lower-level strong\n        # convexity. This improves gradient-penalty convergence by making\n        # the inner problem well-conditioned. Faster gamma ramp (25k vs 30k)\n        # and higher gamma_max=40 amplify the penalty signal while the\n        # lr_decay=1/gamma schedule provides natural annealing.\n        return HypercleanStrategy(\n            method=\"g_pbgd\",\n            lrx=0.5,\n            lry=0.5,\n            gamma_init=0.0,\n            gamma_max=40.0,\n            gamma_argmax_step=25000,\n            outer_itr=60000,\n            reg=0.001,\n            eval_interval=10,\n        )\n    raise ValueError(f\"Unsupported network: {net}\")\n"
7}

1{
2  "op": "replace",
3  "filename": "penalized-bilevel-gradient-descent/mlsbench/custom_strategy.py",
4  "start_line": 216,
5  "end_line": 265,
6  "content": "def get_toy_strategy() -> ToyStrategy:\n    # Proven stable in test #1: alpha=0.02 gives 100% convergence with ~147 steps\n    # (vs 260.71 baseline). gamma=10 maintains strong penalty for feasibility.\n    return ToyStrategy(\n        method=\"v_pbgd\",\n        gams=(10.0,),\n        alpha0=0.2,\n    )\n\n\ndef get_hyperclean_strategy(net: str) -> HypercleanStrategy:\n    if net == \"linear\":\n        # V-PBGD with slightly stronger penalty and faster ramp.\n        # gamma_max=0.25 (vs 0.2) with argmax at step 20k gives more time\n        # at high penalty for better lower-level feasibility.\n        # Faster ramp frontloads penalty signal for better data cleaning.\n        return HypercleanStrategy(\n            method=\"v_pbgd\",\n            lrx=0.1,\n            lry=0.1,\n            lr_inner=0.01,\n            gamma_init=0.0,\n            gamma_max=0.25,\n            gamma_argmax_step=20000,\n            outer_itr=40000,\n            inner_itr=1,\n            reg=0.0,\n            eval_interval=10,\n        )\n    if net == \"mlp\":\n        # G-PBGD with L2 regularization for lower-level strong convexity.\n        # reg=0.001 conditions the inner problem, improving gradient-penalty\n        # convergence. Proven in test #2: 92.64% (vs 92.38% baseline).\n        return HypercleanStrategy(\n            method=\"g_pbgd\",\n            lrx=0.5,\n            lry=0.5,\n            gamma_init=0.0,\n            gamma_max=40.0,\n            gamma_argmax_step=25000,\n            outer_itr=60000,\n            reg=0.001,\n            eval_interval=10,\n        )\n    raise ValueError(f\"Unsupported network: {net}\")\n"
7}