1{
2  "op": "replace",
3  "filename": "custom_boosting.py",
4  "start_line": 147,
5  "end_line": 256,
6  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Loss-Aware Boosting with Focal Residuals and Progressive Annealing.\n\n    Novel contributions:\n    1. Focal pseudo-residuals: residual scaled by (1 - confidence)^gamma,\n       de-emphasizing easy samples while retaining smooth gradient signal.\n    2. Progressive annealing reweighting: blend of AdaBoost exponential and\n       gradient-style uniform weights, with annealing schedule that transitions\n       from aggressive to gentle reweighting across boosting rounds.\n    3. Diversity-aware learner weighting: Newton-like step size with correction\n       for learner-ensemble correlation (diversity bonus for unique learners).\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Focal loss gamma for pseudo-residual scaling\n        self.gamma = 2.0\n        # Progressive annealing: controls reweighting aggression schedule\n        self.anneal_rate = 0.02\n        # L2 regularization for Newton step\n        self.reg_lambda = 1.0\n        # Internal state\n        self._raw_scores = None\n        self._initial_loss = None\n        \n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights.\n\n        Args:\n            n_samples: int — number of training samples.\n\n        Returns:\n            np.ndarray of shape [n_samples] — initial sample weights (should sum to 1).\n        \"\"\"\n        self._raw_scores = np.zeros(n_samples)\n        self._initial_loss = None\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets for the next weak learner to fit.\n\n        Uses focal pseudo-residuals: the negative gradient of the loss\n        scaled by a focal factor (1 - confidence)^gamma that automatically\n        down-weights samples the ensemble is confident about.\n\n        Args:\n            y: np.ndarray [n_samples] — true labels/targets.\n            current_predictions: np.ndarray [n_samples] — ensemble prediction so far\n                (raw scores for classification, values for regression).\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            round_idx: int — current boosting round (0-indexed).\n\n        Returns:\n            np.ndarray [n_samples] — pseudo-targets to fit the weak learner on.\n        \"\"\"\n        if self.task_type == \"classification\":\n            # Negative gradient of log-loss: y - sigmoid(raw_score)\n            probs = self._sigmoid(self._raw_scores)\n            raw_gradient = y - probs\n            \n            # Focal scaling: down-weight confident predictions\n            # Confidence = max(prob, 1-prob) for correct class probability\n            confidence = np.where(y == 1, probs, 1.0 - probs)\n            focal_factor = (1.0 - confidence) ** self.gamma\n            \n            # Combine gradient with focal scaling\n            pseudo_targets = raw_gradient * focal_factor\n            \n        else:\n            # Regression: negative gradient of squared error = residuals\n            raw_residual = y - current_predictions\n            \n            # Compute initial loss for adaptive scaling\n            if self._initial_loss is None:\n                self._initial_loss = np.mean(raw_residual ** 2) + 1e-10\n            \n            # Huber-like focal scaling: reduce influence of outliers\n            # but maintain signal for medium-difficulty samples\n            residual_magnitude = np.abs(raw_residual)\n            # Scale by (1 - normalized_magnitude)^gamma, but clip to avoid negative\n            confidence = 1.0 - (residual_magnitude / \n                               (residual_magnitude + np.sqrt(self._initial_loss)))\n            focal_factor = np.clip(confidence, 0.0, 1.0) ** self.gamma\n            \n            pseudo_targets = raw_residual * (0.5 + 0.5 * focal_factor)\n            \n        return pseudo_targets\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute the weight (alpha) for the newly fitted learner.\n\n        Uses diversity-aware Newton step: the optimal step size in the\n        learner's direction, adjusted to penalize learners highly\n        correlated with previous ensemble predictions (encouraging diversity).\n\n        Args:\n            learner: BaseLearner — the just-fitted weak learner.\n            X: np.ndarray [n_samples, n_features] — training features.\n            y: np.ndarray [n_samples] — true labels/targets.\n            pseudo_targets: np.ndarray [n_samples] — what the learner was fit on.\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            round_idx: int — current boosting round.\n\n        Returns:\n            float — learner weight alpha_t.\n        \"\"\"\n        preds = learner.predict(X)\n        \n        if self.task_type == \"classification\":\n            # Newton step for log-loss\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            \n            # Gradient alignment: correlation between preds and gradient\n            numerator = np.sum(sample_weights * pseudo_targets * preds)\n            denominator = np.sum(sample_weights * hessians * preds ** 2) + self.reg_lambda\n            \n            alpha = numerator / denominator\n            \n            # Diversity correction: penalize learners too correlated with\n            # the current ensemble (they add redundant information)\n            raw_scores_current = self._raw_scores\n            if np.std(raw_scores_current) > 1e-10:\n                ensemble_direction = np.tanh(raw_scores_current)  # [-1, 1]\n                diversity_factor = 1.0 - np.abs(np.corrcoef(preds, ensemble_direction)[0, 1])\n                diversity_factor = np.clip(diversity_factor, 0.3, 1.5)\n                alpha *= diversity_factor\n            \n            return max(alpha, 0.0)\n            \n        else:\n            # Regression: standard Newton step with diversity bonus\n            numerator = np.sum(sample_weights * pseudo_targets * preds)\n            denominator = np.sum(sample_weights * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            \n            # Diversity correction for regression\n            if round_idx > 0 and np.std(preds) > 1e-10:\n                # Estimate ensemble prediction direction from previous learners\n                # Use current_residual as proxy for what ensemble needs\n                residual = y - (self._raw_scores if self._raw_scores is not None \n                               else np.zeros_like(y))\n                if np.std(residual) > 1e-10:\n                    corr = np.corrcoef(preds, residual)[0, 1]\n                    # Reward learners that correlate with what's needed\n                    diversity_bonus = 0.5 + 0.5 * max(0, corr)\n                    alpha *= diversity_bonus\n            \n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights after fitting a learner.\n\n        Uses progressive annealing: a blend of AdaBoost-style exponential\n        reweighting and gradient-style uniform weights. The blend ratio\n        anneals over rounds, starting aggressive (focus on errors) and\n        gradually becoming more uniform (fine-tuning without overfitting).\n\n        Args:\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            learner: BaseLearner — the just-fitted weak learner.\n            X: np.ndarray [n_samples, n_features] — training features.\n            y: np.ndarray [n_samples] — true labels/targets.\n            pseudo_targets: np.ndarray [n_samples] — what the learner was fit on.\n            alpha: float — the learner's weight.\n            round_idx: int — current boosting round.\n\n        Returns:\n            np.ndarray [n_samples] — updated sample weights (should sum to 1).\n        \"\"\"\n        preds = learner.predict(X)\n        \n        # Compute per-sample error\n        if self.task_type == \"classification\":\n            incorrect = (preds != y).astype(float)\n            error_signal = incorrect\n        else:\n            # Normalized error for regression\n            residuals = np.abs(preds - pseudo_targets)\n            max_res = residuals.max() + 1e-10\n            error_signal = residuals / max_res\n        \n        # Progressive annealing schedule\n        # anneal_weight goes from ~1.0 (aggressive) to ~0.0 (uniform)\n        anneal_weight = np.exp(-self.anneal_rate * round_idx)\n        \n        if self.task_type == \"classification\":\n            # AdaBoost-style exponential reweighting\n            reweight_factor = np.exp(alpha * error_signal)\n        else:\n            # For regression, use a softer exponential based on error\n            beta = 0.5  # Base decay factor\n            reweight_factor = np.power(beta, 1.0 - error_signal)\n        \n        # Blend between reweighted and uniform\n        boost_weights = sample_weights * reweight_factor\n        uniform_weights = np.ones_like(sample_weights) / len(sample_weights)\n        \n        # Annealed blend\n        new_weights = anneal_weight * boost_weights + (1.0 - anneal_weight) * uniform_weights\n        \n        return new_weights"
7}

1{
2  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Loss-Aware Boosting with Focal Residuals and Progressive Annealing.\n\n    Novel contributions:\n    1. Focal pseudo-residuals: residual scaled by (1 - confidence)^gamma,\n       de-emphasizing easy samples while retaining smooth gradient signal.\n    2. Progressive annealing reweighting: blend of AdaBoost exponential and\n       gradient-style uniform weights, with annealing schedule that transitions\n       from aggressive to gentle reweighting across boosting rounds.\n    3. Diversity-aware learner weighting: Newton-like step size with correction\n       for learner-ensemble correlation (diversity bonus for unique learners).\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Focal loss gamma for pseudo-residual scaling\n        self.gamma = 2.0\n        # Progressive annealing: controls reweighting aggression schedule\n        self.anneal_rate = 0.02\n        # L2 regularization for Newton step\n        self.reg_lambda = 1.0\n        # Internal state\n        self._raw_scores = None\n        self._initial_loss = None\n\n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights.\n\n        Args:\n            n_samples: int — number of training samples.\n\n        Returns:\n            np.ndarray of shape [n_samples] — initial sample weights (should sum to 1).\n        \"\"\"\n        self._raw_scores = np.zeros(n_samples)\n        self._initial_loss = None\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets for the next weak learner to fit.\n\n        Uses focal pseudo-residuals: the negative gradient of the loss\n        scaled by a focal factor (1 - confidence)^gamma that automatically\n        down-weights samples the ensemble is confident about.\n\n        Args:\n            y: np.ndarray [n_samples] — true labels/targets.\n            current_predictions: np.ndarray [n_samples] — ensemble prediction so far\n                (raw scores for classification, values for regression).\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            round_idx: int — current boosting round (0-indexed).\n\n        Returns:\n            np.ndarray [n_samples] — pseudo-targets to fit the weak learner on.\n        \"\"\"\n        if self.task_type == \"classification\":\n            # Negative gradient of log-loss: y - sigmoid(raw_score)\n            probs = self._sigmoid(self._raw_scores)\n            raw_gradient = y - probs\n\n            # Focal scaling: down-weight confident predictions\n            # Confidence = max(prob, 1-prob) for correct class probability\n            confidence = np.where(y == 1, probs, 1.0 - probs)\n            focal_factor = (1.0 - confidence) ** self.gamma\n\n            # Combine gradient with focal scaling\n            pseudo_targets = raw_gradient * focal_factor\n\n        else:\n            # Regression: negative gradient of squared error = residuals\n            raw_residual = y - current_predictions\n\n            # Compute initial loss for adaptive scaling\n            if self._initial_loss is None:\n                self._initial_loss = np.mean(raw_residual ** 2) + 1e-10\n\n            # Huber-like focal scaling: reduce influence of outliers\n            # but maintain signal for medium-difficulty samples\n            residual_magnitude = np.abs(raw_residual)\n            # Scale by (1 - normalized_magnitude)^gamma, but clip to avoid negative\n            confidence = 1.0 - (residual_magnitude /\n                               (residual_magnitude + np.sqrt(self._initial_loss)))\n            focal_factor = np.clip(confidence, 0.0, 1.0) ** self.gamma\n\n            pseudo_targets = raw_residual * (0.5 + 0.5 * focal_factor)\n\n        return pseudo_targets\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute the weight (alpha) for the newly fitted learner.\n\n        Uses diversity-aware Newton step: the optimal step size in the\n        learner's direction, adjusted to penalize learners highly\n        correlated with previous ensemble predictions (encouraging diversity).\n\n        Args:\n            learner: BaseLearner — the just-fitted weak learner.\n            X: np.ndarray [n_samples, n_features] — training features.\n            y: np.ndarray [n_samples] — true labels/targets.\n            pseudo_targets: np.ndarray [n_samples] — what the learner was fit on.\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            round_idx: int — current boosting round.\n\n        Returns:\n            float — learner weight alpha_t.\n        \"\"\"\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            # Newton step for log-loss\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n\n            # Gradient alignment: correlation between preds and gradient\n            numerator = np.sum(sample_weights * pseudo_targets * preds)\n            denominator = np.sum(sample_weights * hessians * preds ** 2) + self.reg_lambda\n\n            alpha = numerator / denominator\n\n            # Diversity correction: penalize learners too correlated with\n            # the current ensemble (they add redundant information)\n            raw_scores_current = self._raw_scores\n            if np.std(raw_scores_current) > 1e-10:\n                ensemble_direction = np.tanh(raw_scores_current)  # [-1, 1]\n                diversity_factor = 1.0 - np.abs(np.corrcoef(preds, ensemble_direction)[0, 1])\n                diversity_factor = np.clip(diversity_factor, 0.3, 1.5)\n                alpha *= diversity_factor\n\n            return max(alpha, 0.0)\n\n        else:\n            # Regression: standard Newton step with diversity bonus\n            numerator = np.sum(sample_weights * pseudo_targets * preds)\n            denominator = np.sum(sample_weights * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n\n            # Diversity correction for regression\n            if round_idx > 0 and np.std(preds) > 1e-10:\n                # Estimate ensemble prediction direction from previous learners\n                # Use current_residual as proxy for what ensemble needs\n                residual = y - (self._raw_scores if self._raw_scores is not None\n                               else np.zeros_like(y))\n                if np.std(residual) > 1e-10:\n                    corr = np.corrcoef(preds, residual)[0, 1]\n                    # Reward learners that correlate with what's needed\n                    diversity_bonus = 0.5 + 0.5 * max(0, corr)\n                    alpha *= diversity_bonus\n\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights after fitting a learner.\n\n        Uses progressive annealing: a blend of AdaBoost-style exponential\n        reweighting and gradient-style uniform weights. The blend ratio\n        anneals over rounds, starting aggressive (focus on errors) and\n        gradually becoming more uniform (fine-tuning without overfitting).\n\n        Args:\n            sample_weights: np.ndarray [n_samples] — current sample weights.\n            learner: BaseLearner — the just-fitted weak learner.\n            X: np.ndarray [n_samples, n_features] — training features.\n            y: np.ndarray [n_samples] — true labels/targets.\n            pseudo_targets: np.ndarray [n_samples] — what the learner was fit on.\n            alpha: float — the learner's weight.\n            round_idx: int — current boosting round.\n\n        Returns:\n            np.ndarray [n_samples] — updated sample weights (should sum to 1).\n        \"\"\"\n        preds = learner.predict(X)\n\n        # Compute per-sample error\n        if self.task_type == \"classification\":\n            incorrect = (preds != y).astype(float)\n            error_signal = incorrect\n        else:\n            # Normalized error for regression\n            residuals = np.abs(preds - pseudo_targets)\n            max_res = residuals.max() + 1e-10\n            error_signal = residuals / max_res\n\n        # Progressive annealing schedule\n        # anneal_weight goes from ~1.0 (aggressive) to ~0.0 (uniform)\n        anneal_weight = np.exp(-self.anneal_rate * round_idx)\n\n        if self.task_type == \"classification\":\n            # AdaBoost-style exponential reweighting\n            reweight_factor = np.exp(alpha * error_signal)\n        else:\n            # For regression, use a softer exponential based on error\n            beta = 0.5  # Base decay factor\n            reweight_factor = np.power(beta, 1.0 - error_signal)\n\n        # Blend between reweighted and uniform\n        boost_weights = sample_weights * reweight_factor\n        uniform_weights = np.ones_like(sample_weights) / len(sample_weights)\n\n        # Annealed blend\n        new_weights = anneal_weight * boost_weights + (1.0 - anneal_weight) * uniform_weights\n\n        return new_weights",
3  "end_line": 256,
4  "filename": "custom_boosting.py",
5  "op": "replace",
6  "start_line": 147
7}

1{
2  "end_line": 256,
3  "filename": "custom_boosting.py",
4  "op": "replace",
5  "start_line": 147,
6  "content": "class BoostingStrategy:\n    \"\"\"Huber-Boost: Gradient Direction with Adaptive Huber-Loss Reweighting.\n\n    Three key ideas:\n    1. Huber gradient pseudo-targets: Use Huber loss gradients for regression\n       (smooth + robust to outliers) and log-loss gradients for classification.\n    2. Loss-based sample reweighting: Reweight samples proportional to their\n       current loss contribution, smoothly blending AdaBoost's hard-example\n       focus with gradient boosting's uniform fitting.\n    3. Newton step with adaptive clipping: Optimal step size via Newton\n       method, with clipping that adapts based on learner quality.\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Huber threshold parameter (delta)\n        self.huber_delta = 1.0\n        # Reweighting temperature: controls how sharply we focus on hard examples\n        self.reweight_temp = 2.0\n        # L2 regularization for Newton step\n        self.reg_lambda = 1.0\n        # Internal state: raw scores for gradient computation\n        self._raw_scores = None\n\n    def init_weights(self, n_samples):\n        self._raw_scores = np.zeros(n_samples)\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        if self.task_type == \"classification\":\n            # Negative gradient of log-loss: y - sigmoid(raw_score)\n            probs = self._sigmoid(self._raw_scores)\n            return y - probs\n        else:\n            # Huber loss gradient: smooth + robust to outliers\n            residuals = y - current_predictions\n            # For squared error: -dL/dF = residual\n            # Huber modification: clip large residuals\n            abs_res = np.abs(residuals)\n            # Huber scaling: linear for large residuals, quadratic for small\n            huber_scale = np.where(\n                abs_res <= self.huber_delta,\n                1.0,\n                self.huber_delta / abs_res  # clip to delta\n            )\n            return residuals * huber_scale\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            # Newton step for log-loss with hessian = p*(1-p)\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n        else:\n            # Newton step for Huber/squared loss\n            # gradient ~ residual, hessian = 1 (for squared error region)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Loss-based adaptive sample reweighting.\n\n        Reweight samples proportionally to their loss contribution,\n        creating a smooth focus mechanism that bridges AdaBoost (discrete)\n        and gradient boosting (uniform).\n        \"\"\"\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            # Compute log-loss per sample\n            probs = self._sigmoid(self._raw_scores)\n            eps = 1e-10\n            log_loss = -(y * np.log(probs + eps) + (1 - y) * np.log(1 - probs + eps))\n            # Use loss as reweighting signal with temperature\n            loss_weights = np.exp(self.reweight_temp * log_loss / (log_loss.mean() + eps))\n        else:\n            # For regression: use Huber loss per sample\n            residuals = y - (self._raw_scores if self._raw_scores is not None\n                           else np.zeros_like(y))\n            abs_res = np.abs(residuals)\n            huber_loss = np.where(\n                abs_res <= self.huber_delta,\n                0.5 * residuals ** 2,\n                self.huber_delta * (abs_res - 0.5 * self.huber_delta)\n            )\n            # Exponentially reweight by loss\n            loss_weights = np.exp(self.reweight_temp * huber_loss /\n                                  (huber_loss.mean() + eps))\n\n        # Smoothly blend loss-based reweighting with uniform\n        # Early rounds: more uniform (explore), later rounds: more loss-focused\n        round_progress = round_idx / max(self.n_rounds - 1, 1)\n        # Start with moderate focus, increase over time\n        focus_schedule = 0.3 + 0.7 * (1.0 - np.exp(-3.0 * round_progress))\n\n        uniform_w = np.ones_like(sample_weights) / len(sample_weights)\n        new_weights = (1.0 - focus_schedule) * uniform_w + focus_schedule * (\n            sample_weights * loss_weights / (loss_weights.mean() + 1e-10)\n        )\n\n        return new_weights"
7}

1{
2  "end_line": 256,
3  "filename": "scikit-learn/custom_boosting.py",
4  "op": "replace",
5  "start_line": 147,
6  "content": "class BoostingStrategy:\n    \"\"\"Huber-Boost: Gradient Direction with Adaptive Huber-Loss Reweighting.\n\n    Three key ideas:\n    1. Huber gradient pseudo-targets: Use Huber loss gradients for regression\n       (smooth + robust to outliers) and log-loss gradients for classification.\n    2. Loss-based sample reweighting: Reweight samples proportional to their\n       current loss contribution, smoothly blending AdaBoost's hard-example\n       focus with gradient boosting's uniform fitting.\n    3. Newton step with adaptive clipping: Optimal step size via Newton\n       method, with clipping that adapts based on learner quality.\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Huber threshold parameter (delta)\n        self.huber_delta = 1.0\n        # Reweighting temperature: controls how sharply we focus on hard examples\n        self.reweight_temp = 2.0\n        # L2 regularization for Newton step\n        self.reg_lambda = 1.0\n        # Internal state: raw scores for gradient computation\n        self._raw_scores = None\n\n    def init_weights(self, n_samples):\n        self._raw_scores = np.zeros(n_samples)\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        if self.task_type == \"classification\":\n            # Negative gradient of log-loss: y - sigmoid(raw_score)\n            probs = self._sigmoid(self._raw_scores)\n            return y - probs\n        else:\n            # Huber loss gradient: smooth + robust to outliers\n            residuals = y - current_predictions\n            abs_res = np.abs(residuals)\n            huber_scale = np.where(\n                abs_res <= self.huber_delta,\n                1.0,\n                self.huber_delta / abs_res\n            )\n            return residuals * huber_scale\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n        else:\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            eps = 1e-10\n            log_loss = -(y * np.log(probs + eps) + (1 - y) * np.log(1 - probs + eps))\n            loss_weights = np.exp(self.reweight_temp * log_loss / (log_loss.mean() + eps))\n        else:\n            residuals = y - current_predictions\n            abs_res = np.abs(residuals)\n            huber_loss = np.where(\n                abs_res <= self.huber_delta,\n                0.5 * residuals ** 2,\n                self.huber_delta * (abs_res - 0.5 * self.huber_delta)\n            )\n            loss_weights = np.exp(self.reweight_temp * huber_loss /\n                                  (huber_loss.mean() + 1e-10))\n\n        round_progress = round_idx / max(self.n_rounds - 1, 1)\n        focus_schedule = 0.3 + 0.7 * (1.0 - np.exp(-3.0 * round_progress))\n\n        uniform_w = np.ones_like(sample_weights) / len(sample_weights)\n        new_weights = (1.0 - focus_schedule) * uniform_w + focus_schedule * (\n            sample_weights * loss_weights / (loss_weights.mean() + 1e-10)\n        )\n\n        return new_weights"
7}

1{
2  "content": "    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        preds = learner.predict(X)\n\n        # Update raw scores for tracking ensemble predictions\n        preds = learner.predict(X)\n        self._raw_scores += self.learning_rate * alpha * preds\n\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            eps = 1e-10\n            log_loss = -(y * np.log(probs + eps) + (1 - y) * np.log(1 - probs + eps))\n            loss_weights = np.exp(self.reweight_temp * log_loss / (log_loss.mean() + eps))\n        else:\n            residuals = y - self._raw_scores\n            abs_res = np.abs(residuals)\n            huber_loss = np.where(\n                abs_res <= self.huber_delta,\n                0.5 * residuals ** 2,\n                self.huber_delta * (abs_res - 0.5 * self.huber_delta)\n            )\n            loss_weights = np.exp(self.reweight_temp * huber_loss /\n                                  (huber_loss.mean() + 1e-10))\n\n        round_progress = round_idx / max(self.n_rounds - 1, 1)\n        focus_schedule = 0.3 + 0.7 * (1.0 - np.exp(-3.0 * round_progress))\n\n        uniform_w = np.ones_like(sample_weights) / len(sample_weights)\n        new_weights = (1.0 - focus_schedule) * uniform_w + focus_schedule * (\n            sample_weights * loss_weights / (loss_weights.mean() + 1e-10)\n        )\n\n        return new_weights",
3  "end_line": 242,
4  "filename": "scikit-learn/custom_boosting.py",
5  "op": "replace",
6  "start_line": 210
7}

1{
2  "content": "    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        preds = learner.predict(X)\n\n        # Update raw scores for tracking ensemble predictions\n        self._raw_scores += self.learning_rate * alpha * preds\n\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            eps = 1e-10\n            log_loss = -(y * np.log(probs + eps) + (1 - y) * np.log(1 - probs + eps))\n            loss_weights = np.exp(self.reweight_temp * log_loss / (log_loss.mean() + eps))\n        else:\n            residuals = y - self._raw_scores\n            abs_res = np.abs(residuals)\n            huber_loss = np.where(\n                abs_res <= self.huber_delta,\n                0.5 * residuals ** 2,\n                self.huber_delta * (abs_res - 0.5 * self.huber_delta)\n            )\n            loss_weights = np.exp(self.reweight_temp * huber_loss /\n                                  (huber_loss.mean() + 1e-10))\n\n        round_progress = round_idx / max(self.n_rounds - 1, 1)\n        focus_schedule = 0.3 + 0.7 * (1.0 - np.exp(-3.0 * round_progress))\n\n        uniform_w = np.ones_like(sample_weights) / len(sample_weights)\n        new_weights = (1.0 - focus_schedule) * uniform_w + focus_schedule * (\n            sample_weights * loss_weights / (loss_weights.mean() + 1e-10)\n        )\n\n        return new_weights",
3  "end_line": 242,
4  "filename": "scikit-learn/custom_boosting.py",
5  "op": "replace",
6  "start_line": 210
7}

1{
2  "content": "    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            return y - probs\n        else:\n            residuals = y - current_predictions\n            abs_res = np.abs(residuals)\n            scale = np.median(abs_res) + 1e-10\n            huber_scale = np.where(\n                abs_res <= 1.35 * scale,\n                1.0,\n                1.35 * scale / abs_res\n            )\n            return residuals * huber_scale",
3  "end_line": 193,
4  "filename": "scikit-learn/custom_boosting.py",
5  "op": "replace",
6  "start_line": 178
7}

1{
2  "content": "class BoostingStrategy:\n    \"\"\"AdaHuber: Adaptive Huber Gradient Boosting with Loss-Aware Reweighting.\n\n    Three key ideas:\n    1. Adaptive Huber gradients for pseudo-targets: Huber scale estimated from\n       Median Absolute Deviation (MAD) of residuals, making it scale-invariant.\n    2. Newton-style learner weighting with L2 regularization and alignment\n       check to ensure learners contribute positively.\n    3. Exponential loss-aware sample reweighting: weight samples proportionally\n       to their current loss, naturally focusing on hard examples while\n       maintaining enough diversity from the gradient signal.\n    \"\"\"\n\n    def __init__(self, config):\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Huber MAD multiplier for scale estimation\n        self.huber_k = 1.35\n        # Reweighting temperature\n        self.reweight_temp = 1.5\n        # L2 regularization for Newton step\n        self.reg_lambda = 1.0\n        # Internal state\n        self._raw_scores = None\n        self._cum_preds = None\n\n    def init_weights(self, n_samples):\n        self._raw_scores = np.zeros(n_samples)\n        self._cum_preds = None\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            return y - probs\n        else:\n            # Track cumulative predictions for internal use\n            if self._cum_preds is None:\n                self._cum_preds = current_predictions.copy()\n            residuals = y - current_predictions\n            # Adaptive Huber scale via MAD\n            abs_res = np.abs(residuals)\n            mad = np.median(abs_res) + 1e-10\n            huber_scale = np.where(\n                abs_res <= self.huber_k * mad,\n                1.0,\n                self.huber_k * mad / abs_res\n            )\n            return residuals * huber_scale\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        preds = learner.predict(X)\n\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n        else:\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / (denominator + 1e-10)\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        preds = learner.predict(X)\n\n        # Update internal states\n        self._raw_scores += self.learning_rate * alpha * preds\n        if self.task_type == \"regression\":\n            if self._cum_preds is None:\n                self._cum_preds = np.full(preds.shape, y.mean())\n            self._cum_preds += self.learning_rate * alpha * preds\n\n        # Compute per-sample loss\n        if self.task_type == \"classification\":\n            probs = self._sigmoid(self._raw_scores)\n            eps = 1e-10\n            loss = -(y * np.log(probs + eps) + (1 - y) * np.log(1 - probs + eps))\n        else:\n            residuals = y - self._cum_preds\n            abs_res = np.abs(residuals)\n            mad = np.median(abs_res) + 1e-10\n            loss = np.where(\n                abs_res <= self.huber_k * mad,\n                0.5 * residuals ** 2,\n                self.huber_k * mad * (abs_res - 0.5 * self.huber_k * mad)\n            )\n\n        # Reweight by exp(loss / mean_loss * temp)\n        mean_loss = loss.mean() + 1e-10\n        loss_weights = np.exp(self.reweight_temp * loss / mean_loss)\n\n        # Normalize\n        new_weights = sample_weights * loss_weights\n        return new_weights",
3  "end_line": 239,
4  "filename": "scikit-learn/custom_boosting.py",
5  "op": "replace",
6  "start_line": 147
7}