1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 256,
6  "content": "class BoostingStrategy:\n    \"\"\"Robust Adaptive Newton Boosting (RAN-Boost).\n\n    Combines three algorithmic innovations for improved gradient boosting:\n\n    1. WINSORIZED GRADIENT TARGETS: For regression, pseudo-residuals are\n       percentile-clipped each round. Threshold adapts to dataset size\n       (pct ~ 1/sqrt(n)), giving strong robustness for small datasets\n       and nearly unclipped gradients for large ones.\n\n    2. DECAYING NEWTON REGULARIZATION: L2 penalty on per-round Newton step\n       follows quadratic decay: lam(t) = lam0*(1-t/T)^2 + eps. Heavy early\n       regularization prevents overfitting when ensemble is weak; minimal\n       late regularization enables precise fine-tuning.\n\n    3. POST-CORRECTION RESIDUAL FOCUS: Sample weights are mildly adjusted\n       based on residuals remaining AFTER each round's correction -- a\n       forward-looking reweighting that focuses tree capacity on\n       systematically underfit regions without AdaBoost.R2's feedback loops.\n\n    For classification: bounded AdaBoost (exponential reweighting with\n    weight capping for robustness to label noise).\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        self._current_preds = None\n        n_samples = config[\"n_samples\"]\n        n_features = config[\"n_features\"]\n        # Adaptive regularization: scales with problem dimension\n        self.reg_lambda_init = np.sqrt(n_features) + 0.5\n        self.reg_lambda_min = 0.1\n        # Winsorization: adapts to dataset size (stronger for small n)\n        self.clip_pct = max(0.5, min(5.0, 50.0 / np.sqrt(n_samples)))\n        # Mild residual-proportional reweighting\n        self.focus_strength = 0.2\n        # Classification weight cap (multiples of uniform weight)\n        self.weight_cap = 20.0\n\n    def init_weights(self, n_samples):\n        \"\"\"Uniform initialization.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Original labels (classification) or winsorized residuals (regression).\"\"\"\n        self._current_preds = current_predictions.copy()\n        if self.task_type == \"classification\":\n            return y\n        residuals = y - current_predictions\n        lo = np.percentile(residuals, self.clip_pct)\n        hi = np.percentile(residuals, 100 - self.clip_pct)\n        return np.clip(residuals, lo, hi) if lo < hi else residuals\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"AdaBoost alpha (classification) or regularized Newton step (regression).\"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"classification\":\n            incorrect = (preds != y).astype(float)\n            w_err = np.dot(sample_weights, incorrect) / sample_weights.sum()\n            w_err = np.clip(w_err, 1e-10, 1.0 - 1e-10)\n            alpha = self.learning_rate * 0.5 * np.log((1.0 - w_err) / w_err)\n            return max(alpha, 0.0)\n        # Quadratic decay schedule: strong regularization early, weak late\n        progress = round_idx / max(self.n_rounds - 1, 1)\n        lam = self.reg_lambda_init * (1.0 - progress) ** 2 + self.reg_lambda_min\n        num = np.sum(pseudo_targets * preds)\n        den = np.sum(preds ** 2) + lam\n        return max(num / den, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Bounded AdaBoost (classification) or residual focus (regression).\"\"\"\n        preds = learner.predict(X)\n        n = len(sample_weights)\n        if self.task_type == \"classification\":\n            # Exponential reweighting with weight capping\n            incorrect = (preds != y).astype(float)\n            sample_weights = sample_weights * np.exp(alpha * incorrect)\n            sample_weights = np.minimum(sample_weights, self.weight_cap / n)\n        else:\n            # Forward-looking: reweight by post-correction residual magnitude\n            updated = self._current_preds + alpha * self.learning_rate * preds\n            res = np.abs(y - updated)\n            scale = np.percentile(res, 95) + 1e-10\n            norm_res = np.clip(res / scale, 0.0, 2.0)\n            sample_weights = sample_weights * (1.0 + self.focus_strength * norm_res)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights\n\n"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 242,
6  "content": "class BoostingStrategy:\n    \"\"\"Robust Adaptive Newton Boosting (RAN-Boost).\n\n    Novel algorithmic contributions over standard gradient boosting:\n\n    1. WINSORIZED GRADIENT TARGETS: For regression, pseudo-residuals are\n       percentile-clipped each round. The clip threshold adapts to dataset\n       size (pct ~ 1/sqrt(n)), giving strong outlier robustness for small\n       noisy datasets and nearly unclipped gradients for large ones.\n       This effectively optimizes a Winsorized squared loss.\n\n    2. SIZE-ADAPTIVE DECAYING REGULARIZATION: Newton step L2 penalty adapts\n       to dataset size (stronger for small n where overfitting risk is high,\n       matching XGBoost defaults for large n) and decays linearly over\n       rounds. Heavy early regularization prevents overfitting when the\n       ensemble is weak; reduced late regularization enables fine-tuning.\n\n    3. NON-COMPOUNDING RESIDUAL FOCUS: Each round, sample weights are reset\n       to uniform and recomputed based on post-correction residual magnitudes.\n       Unlike multiplicative schemes (AdaBoost.R2) that compound exponentially\n       and can create degenerate weight distributions, this bounded scheme\n       provides fresh, stable focus on systematically underfit regions.\n       Focus strength adapts to sqrt(n) to avoid noise amplification on\n       large datasets.\n\n    For classification: standard AdaBoost (proven optimal for binary tasks\n    with exponential loss, kept unmodified).\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        self._current_preds = None\n        n_samples = config[\"n_samples\"]\n        # Winsorization percentile: stronger for small datasets\n        self.clip_pct = max(0.5, min(5.0, 50.0 / np.sqrt(n_samples)))\n        # Regularization: stronger for small datasets, decays over rounds\n        self.reg_lambda_init = max(1.0, 3.0 - n_samples / 500.0)\n        self.reg_lambda_min = 0.5\n        # Non-compounding focus: weaker for large datasets\n        self.focus_strength = min(0.3, 5.0 / np.sqrt(n_samples))\n\n    def init_weights(self, n_samples):\n        \"\"\"Uniform initialization.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Original labels (classification) or winsorized residuals (regression).\"\"\"\n        self._current_preds = current_predictions.copy()\n        if self.task_type == \"classification\":\n            return y  # AdaBoost: fit on original labels\n        # Regression: winsorized residuals for robustness\n        residuals = y - current_predictions\n        lo = np.percentile(residuals, self.clip_pct)\n        hi = np.percentile(residuals, 100 - self.clip_pct)\n        return np.clip(residuals, lo, hi) if lo < hi else residuals\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"AdaBoost alpha (classification) or Newton step with decaying\n        regularization (regression).\"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"classification\":\n            # Standard AdaBoost alpha (unchanged)\n            incorrect = (preds != y).astype(float)\n            w_err = np.dot(sample_weights, incorrect) / sample_weights.sum()\n            w_err = np.clip(w_err, 1e-10, 1.0 - 1e-10)\n            return self.learning_rate * 0.5 * np.log((1.0 - w_err) / w_err)\n        # Regression: Newton step with linearly decaying regularization\n        progress = round_idx / max(self.n_rounds - 1, 1)\n        lam = (self.reg_lambda_init * (1.0 - progress)\n               + self.reg_lambda_min * progress)\n        num = np.sum(pseudo_targets * preds)\n        den = np.sum(preds ** 2) + lam\n        return max(num / den, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Standard AdaBoost reweighting (classification) or non-compounding\n        residual focus (regression).\"\"\"\n        preds = learner.predict(X)\n        n = len(sample_weights)\n        if self.task_type == \"classification\":\n            # Standard AdaBoost exponential reweighting\n            incorrect = (preds != y).astype(float)\n            sample_weights = sample_weights * np.exp(alpha * incorrect)\n        else:\n            # Non-compounding: reset to uniform and apply fresh focus\n            # based on post-correction residual magnitude\n            updated = self._current_preds + alpha * self.learning_rate * preds\n            res = np.abs(y - updated)\n            scale = np.percentile(res, 95) + 1e-10\n            norm_res = np.clip(res / scale, 0.0, 2.0)\n            sample_weights = np.ones(n) / n  # Reset: prevents compounding\n            sample_weights = sample_weights * (1.0 + self.focus_strength * norm_res)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights\n\n"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 249,
6  "content": "class BoostingStrategy:\n    \"\"\"Progressive Winsorization Newton Boosting (PWN-Boost).\n\n    Novel contributions for improved gradient boosting:\n\n    1. PROGRESSIVE WINSORIZATION: Residual pseudo-targets are percentile-\n       clipped with a threshold that grows over boosting rounds:\n         clip_pct(t) = base * (1 + growth * t/T)\n       Early rounds preserve learning signal with mild clipping; late rounds\n       clip aggressively, treating remaining residuals as increasingly noisy.\n       This provides implicit regularization analogous to a learning rate\n       schedule but operating on the target space. Base threshold adapts to\n       dataset size (~1/sqrt(n)), so large datasets get negligible clipping.\n\n    2. SIZE-ADAPTIVE NEWTON REGULARIZATION: L2 penalty on the Newton step\n       scales inversely with dataset size, providing heavier regularization\n       for small datasets where overfitting risk is highest.\n\n    3. NON-COMPOUNDING RESIDUAL FOCUS: For small datasets, sample weights\n       are reset each round and recomputed from post-correction residual\n       magnitudes. Bounded by construction (max ratio ~1.4x), this avoids\n       exponential weight explosion of multiplicative schemes while gently\n       concentrating tree capacity on systematically underfit regions.\n\n    Classification: standard AdaBoost (unchanged).\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        self._current_preds = None\n        n_samples = config[\"n_samples\"]\n        # Progressive winsorization: adapts to dataset size\n        # Large datasets get ~0 clipping; small datasets get meaningful clipping\n        self.base_clip_pct = max(0.0, min(5.0,\n                                 50.0 / np.sqrt(n_samples) - 0.3))\n        self.clip_growth = max(0.0, 3.0 - n_samples / 1000.0)\n        # Newton regularization: stronger for small datasets\n        self.reg_lambda = max(1.0, 3.0 - n_samples / 500.0)\n        # Non-compounding focus for small datasets only\n        self.use_focus = (n_samples < 2000)\n        self.focus_strength = 0.2\n\n    def init_weights(self, n_samples):\n        \"\"\"Uniform initialization.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def compute_targets(self, y, current_predictions, sample_weights,\n                        round_idx):\n        \"\"\"Original labels (classification) or progressively winsorized\n        residuals (regression).\"\"\"\n        self._current_preds = current_predictions.copy()\n        if self.task_type == \"classification\":\n            return y\n        residuals = y - current_predictions\n        # Progressive clipping: increases over rounds\n        progress = round_idx / max(self.n_rounds - 1, 1)\n        clip_pct = min(25.0,\n                       self.base_clip_pct * (1.0 + self.clip_growth * progress))\n        if clip_pct > 0.01:\n            lo = np.percentile(residuals, clip_pct)\n            hi = np.percentile(residuals, 100.0 - clip_pct)\n            if lo < hi:\n                return np.clip(residuals, lo, hi)\n        return residuals\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"AdaBoost alpha (classification) or Newton step (regression).\"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"classification\":\n            incorrect = (preds != y).astype(float)\n            w_err = np.dot(sample_weights, incorrect) / sample_weights.sum()\n            w_err = np.clip(w_err, 1e-10, 1.0 - 1e-10)\n            return self.learning_rate * 0.5 * np.log(\n                (1.0 - w_err) / w_err)\n        num = np.sum(pseudo_targets * preds)\n        den = np.sum(preds ** 2) + self.reg_lambda\n        return max(num / den, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"AdaBoost reweighting (classification) or non-compounding residual\n        focus (regression, small datasets only).\"\"\"\n        if self.task_type == \"classification\":\n            preds = learner.predict(X)\n            incorrect = (preds != y).astype(float)\n            sample_weights = sample_weights * np.exp(alpha * incorrect)\n        elif self.use_focus:\n            preds = learner.predict(X)\n            n = len(sample_weights)\n            updated = self._current_preds + alpha * self.learning_rate * preds\n            res = np.abs(y - updated)\n            scale = np.percentile(res, 95) + 1e-10\n            norm_res = np.clip(res / scale, 0.0, 2.0)\n            sample_weights = np.ones(n) / n  # Reset: no compounding\n            sample_weights *= (1.0 + self.focus_strength * norm_res)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights\n\n"
7}

1{
2  "n": 3
3}