1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 256,
6  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Gradient Boosting with Confidence-Weighted Updates (AGBCW).\n\n    Combines gradient-based pseudo-targets, second-order learner weights,\n    and adaptive sample reweighting based on confidence (margin for classification,\n    normalized residuals for regression). Uses round-dependent temperature to\n    focus more on hard samples in later rounds.\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Regularization parameter for Newton step (like XGBoost lambda)\n        self.reg_lambda = 1.0\n        # Base temperature for exponential weighting (decays over rounds)\n        self.temperature_base = 2.0\n\n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights uniformly.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        \"\"\"Sigmoid function with clipping for numerical stability.\"\"\"\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets as negative gradient of loss.\n\n        For regression: negative gradient of squared loss = residuals.\n        For classification: negative gradient of logistic loss = y - p.\n        \"\"\"\n        if self.task_type == \"regression\":\n            # residuals\n            return y - current_predictions\n        else:\n            # classification: gradient of logistic loss\n            # current_predictions are raw scores (before sigmoid)\n            probs = self._sigmoid(current_predictions)\n            return y - probs\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute learner weight via regularized Newton step.\n\n        For regression: alpha = sum(g * h) / (sum(h^2) + lambda)\n          where g = pseudo_targets (residuals), h = learner predictions,\n          and hessian = 1 (for squared loss).\n        For classification: same but hessian = p*(1-p).\n        \"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"regression\":\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n        else:\n            # classification: need Hessians = p*(1-p) evaluated at current raw scores\n            # current raw scores are not directly available; we can approximate\n            # using the fact that pseudo_targets = y - p, and p = sigmoid(F).\n            # But we don't have F. However, we can compute p from y and pseudo_targets:\n            # p = y - pseudo_targets. However pseudo_targets could be negative.\n            # Better: we need raw scores F to compute p. We can compute F by\n            # inverting sigmoid: F = log(p/(1-p)). But p is unknown.\n            # Alternative: approximate Hessian using constant? Use 0.25 as upper bound.\n            # Instead, we can compute hessian using the fact that we have current_predictions\n            # in compute_targets, but we don't have them here. We'll store them as state.\n            # Let's store current raw scores in a separate variable.\n            # Since we don't have current_predictions in this method, we'll store them\n            # as instance variable updated each round.\n            if not hasattr(self, '_raw_scores'):\n                self._raw_scores = np.zeros(X.shape[0])\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights based on confidence and round.\n\n        For regression:\n            weight update ∝ exp(temperature * normalized_absolute_error)\n            where normalized_absolute_error = |residual - pred| / max_error.\n        For classification:\n            weight update ∝ exp(temperature * confidence_score)\n            confidence_score = gradient_magnitude * exp(-margin)\n            margin = (2*y-1) * raw_score.\n        Temperature decays linearly from temperature_base to 1.0 over rounds.\n        \"\"\"\n        preds = learner.predict(X)\n        # Compute temperature decay: starts high, ends low\n        # So early rounds focus less on hard samples, later rounds focus more.\n        temperature = self.temperature_base - (self.temperature_base - 1.0) * (round_idx / self.n_rounds)\n\n        if self.task_type == \"regression\":\n            # pseudo_targets are residuals (y - current_pred)\n            # preds are fitted residuals\n            errors = np.abs(pseudo_targets - preds)  # absolute error of residual fit\n            if errors.max() > 0:\n                errors = errors / errors.max()  # normalize to [0,1]\n            weight_update = np.exp(temperature * errors)\n        else:\n            # classification\n            # Compute raw scores for margin: need current raw scores.\n            # We'll store raw scores as instance variable and update after weight update.\n            if not hasattr(self, '_raw_scores'):\n                self._raw_scores = np.zeros(X.shape[0])\n            y_signed = 2 * y - 1\n            margin = y_signed * self._raw_scores\n            # gradient magnitude = |pseudo_targets|\n            grad_mag = np.abs(pseudo_targets)\n            # confidence score: high grad_mag + low margin => high weight\n            score = grad_mag * np.exp(-margin)\n            if score.max() > 0:\n                score = score / score.max()\n            weight_update = np.exp(temperature * score)\n            # Update raw scores with new learner contribution\n            self._raw_scores += self.learning_rate * alpha * preds\n\n        sample_weights = sample_weights * weight_update\n        # Normalize\n        sample_weights = np.clip(sample_weights, 1e-10, None)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 280,
6  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Gradient Boosting with Confidence-Weighted Updates (AGBCW).\n\n    Combines gradient-based pseudo-targets, second-order learner weights,\n    and adaptive sample reweighting based on confidence (margin for classification,\n    normalized residuals for regression). Uses round-dependent temperature to\n    focus more on hard samples in later rounds.\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Regularization parameter for Newton step (like XGBoost lambda)\n        self.reg_lambda = 1.0\n        # Base temperature for exponential weighting (decays over rounds)\n        self.temperature_base = 2.0\n        # Smoothing factor for sample weight updates\n        self.smoothing = 0.1\n\n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights uniformly.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        \"\"\"Sigmoid function with clipping for numerical stability.\"\"\"\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets as negative gradient of loss.\n\n        For regression: negative gradient of squared loss = residuals.\n        For classification: negative gradient of logistic loss = y - p.\n        \"\"\"\n        # Store current raw scores for Hessian computation in learner weight\n        self._raw_scores = current_predictions\n        if self.task_type == \"regression\":\n            # residuals\n            return y - current_predictions\n        else:\n            # classification: gradient of logistic loss\n            # current_predictions are raw scores (before sigmoid)\n            probs = self._sigmoid(current_predictions)\n            return y - probs\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute learner weight via regularized Newton step.\n\n        For regression: alpha = sum(g * h) / (sum(h^2) + lambda)\n          where g = pseudo_targets (residuals), h = learner predictions,\n          and hessian = 1 (for squared loss).\n        For classification: same but hessian = p*(1-p).\n        \"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"regression\":\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n        else:\n            # classification: Hessians = p*(1-p) evaluated at current raw scores\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights based on confidence and round.\n\n        For regression:\n            weight update ∝ exp(temperature * normalized_absolute_error)\n            where normalized_absolute_error = |residual - pred| / max_error.\n        For classification:\n            weight update ∝ exp(temperature * confidence_score)\n            confidence_score = gradient_magnitude * exp(-margin)\n            margin = (2*y-1) * raw_score.\n        Temperature decays linearly from temperature_base to 1.0 over rounds.\n        \"\"\"\n        preds = learner.predict(X)\n        # Compute temperature decay: starts high, ends low\n        # So early rounds focus less on hard samples, later rounds focus more.\n        temperature = self.temperature_base - (self.temperature_base - 1.0) * (round_idx / self.n_rounds)\n\n        if self.task_type == \"regression\":\n            # pseudo_targets are residuals (y - current_pred)\n            # preds are fitted residuals\n            errors = np.abs(pseudo_targets - preds)  # absolute error of residual fit\n            if errors.max() > 0:\n                errors = errors / errors.max()  # normalize to [0,1]\n            # Add small smoothing to avoid extreme weights\n            errors = errors + self.smoothing\n            weight_update = np.exp(temperature * errors)\n        else:\n            # classification\n            # Compute margin using raw scores stored in compute_targets\n            y_signed = 2 * y - 1\n            margin = y_signed * self._raw_scores\n            # gradient magnitude = |pseudo_targets|\n            grad_mag = np.abs(pseudo_targets)\n            # confidence score: high grad_mag + low margin => high weight\n            score = grad_mag * np.exp(-margin)\n            if score.max() > 0:\n                score = score / score.max()\n            score = score + self.smoothing\n            weight_update = np.exp(temperature * score)\n\n        sample_weights = sample_weights * weight_update\n        # Normalize with clipping\n        sample_weights = np.clip(sample_weights, 1e-10, None)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 268,
6  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Gradient Boosting with Confidence-Weighted Updates (AGBCW).\n\n    Combines gradient-based pseudo-targets, second-order learner weights,\n    and adaptive sample reweighting based on confidence (margin for classification,\n    normalized residuals for regression). Uses round-dependent temperature to\n    focus more on hard samples in later rounds.\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Regularization parameter for Newton step (like XGBoost lambda)\n        self.reg_lambda = 0.1\n        # Base temperature for exponential weighting (decays over rounds)\n        self.temperature_base = 1.5\n        # Smoothing factor for sample weight updates\n        self.smoothing = 0.2\n        # Weight update exponent (1 for linear, 2 for quadratic)\n        self.exponent = 1.0\n\n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights uniformly.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        \"\"\"Sigmoid function with clipping for numerical stability.\"\"\"\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets as negative gradient of loss.\n\n        For regression: negative gradient of squared loss = residuals.\n        For classification: negative gradient of logistic loss = y - p.\n        \"\"\"\n        # Store current raw scores for Hessian computation in learner weight\n        self._raw_scores = current_predictions\n        if self.task_type == \"regression\":\n            # residuals\n            return y - current_predictions\n        else:\n            # classification: gradient of logistic loss\n            # current_predictions are raw scores (before sigmoid)\n            probs = self._sigmoid(current_predictions)\n            return y - probs\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute learner weight via regularized Newton step.\n\n        For regression: alpha = sum(g * h) / (sum(h^2) + lambda)\n          where g = pseudo_targets (residuals), h = learner predictions,\n          and hessian = 1 (for squared loss).\n        For classification: same but hessian = p*(1-p).\n        \"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"regression\":\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n        else:\n            # classification: Hessians = p*(1-p) evaluated at current raw scores\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights based on confidence and round.\n\n        For regression:\n            weight update ∝ (smoothing + normalized_absolute_error)^exponent\n            where normalized_absolute_error = |residual - pred| / (std_dev + eps).\n        For classification:\n            weight update ∝ exp(temperature * confidence_score)\n            confidence_score = gradient_magnitude * exp(-margin)\n            margin = (2*y-1) * raw_score.\n        Temperature decays linearly from temperature_base to 1.0 over rounds.\n        \"\"\"\n        preds = learner.predict(X)\n        # Compute temperature decay: starts high, ends low\n        # So early rounds focus less on hard samples, later rounds focus more.\n        temperature = self.temperature_base - (self.temperature_base - 1.0) * (round_idx / self.n_rounds)\n\n        if self.task_type == \"regression\":\n            # pseudo_targets are residuals (y - current_pred)\n            # preds are fitted residuals\n            errors = np.abs(pseudo_targets - preds)  # absolute error of residual fit\n            # Normalize by standard deviation of errors (robust)\n            eps = 1e-8\n            if errors.std() > eps:\n                errors = errors / (errors.std() + eps)\n            else:\n                errors = errors / (errors.max() + eps)\n            # Clip errors to reasonable range\n            errors = np.clip(errors, 0, 5)\n            # Weight update: polynomial in errors (linear by default)\n            weight_update = (self.smoothing + errors) ** self.exponent\n        else:\n            # classification\n            # Compute margin using raw scores stored in compute_targets\n            y_signed = 2 * y - 1\n            margin = y_signed * self._raw_scores\n            # gradient magnitude = |pseudo_targets|\n            grad_mag = np.abs(pseudo_targets)\n            # confidence score: high grad_mag + low margin => high weight\n            score = grad_mag * np.exp(-margin)\n            if score.max() > 0:\n                score = score / score.max()\n            score = score + self.smoothing\n            weight_update = np.exp(temperature * score)\n\n        sample_weights = sample_weights * weight_update\n        # Normalize with clipping\n        sample_weights = np.clip(sample_weights, 1e-10, None)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_boosting.py",
4  "start_line": 147,
5  "end_line": 275,
6  "content": "class BoostingStrategy:\n    \"\"\"Adaptive Second-Order Boosting with Hybrid Weighting (ASBHW).\n\n    Combines second-order learner weights (Newton step with regularization)\n    and adaptive sample reweighting:\n    - For regression: AdaBoost.R2-style weight reduction for well-predicted samples.\n    - For classification: confidence-weighted exponential updates based on margin.\n    \"\"\"\n\n    def __init__(self, config):\n        \"\"\"Initialize the boosting strategy.\n\n        Args:\n            config: dict with keys n_samples, n_features, n_rounds,\n                    task_type, learning_rate, dataset.\n        \"\"\"\n        self.config = config\n        self.task_type = config[\"task_type\"]\n        self.n_rounds = config[\"n_rounds\"]\n        self.learning_rate = config[\"learning_rate\"]\n        # Regularization parameter for Newton step (like XGBoost lambda)\n        self.reg_lambda = 0.1\n        # Temperature for classification weight update (decays over rounds)\n        self.temperature_base = 1.5\n\n    def init_weights(self, n_samples):\n        \"\"\"Initialize sample weights uniformly.\"\"\"\n        return np.ones(n_samples) / n_samples\n\n    def _sigmoid(self, x):\n        \"\"\"Sigmoid function with clipping for numerical stability.\"\"\"\n        return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))\n\n    def compute_targets(self, y, current_predictions, sample_weights, round_idx):\n        \"\"\"Compute pseudo-targets as negative gradient of loss.\n\n        For regression: negative gradient of squared loss = residuals.\n        For classification: negative gradient of logistic loss = y - p.\n        \"\"\"\n        # Store current raw scores for Hessian computation in learner weight\n        self._raw_scores = current_predictions\n        if self.task_type == \"regression\":\n            # residuals\n            return y - current_predictions\n        else:\n            # classification: gradient of logistic loss\n            # current_predictions are raw scores (before sigmoid)\n            probs = self._sigmoid(current_predictions)\n            return y - probs\n\n    def compute_learner_weight(self, learner, X, y, pseudo_targets,\n                                sample_weights, round_idx):\n        \"\"\"Compute learner weight via regularized Newton step.\n\n        For regression: alpha = sum(g * h) / (sum(h^2) + lambda)\n          where g = pseudo_targets (residuals), h = learner predictions,\n          and hessian = 1 (for squared loss).\n        For classification: same but hessian = p*(1-p).\n        \"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"regression\":\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n        else:\n            # classification: Hessians = p*(1-p) evaluated at current raw scores\n            probs = self._sigmoid(self._raw_scores)\n            hessians = probs * (1.0 - probs)\n            numerator = np.sum(pseudo_targets * preds)\n            denominator = np.sum(hessians * preds ** 2) + self.reg_lambda\n            alpha = numerator / denominator\n            return max(alpha, 0.0)\n\n    def update_weights(self, sample_weights, learner, X, y, pseudo_targets,\n                       alpha, round_idx):\n        \"\"\"Update sample weights after fitting a learner.\n\n        Regression: AdaBoost.R2-style weight reduction for well-predicted samples.\n        Classification: exponential weighting based on margin and gradient magnitude.\n        \"\"\"\n        preds = learner.predict(X)\n        if self.task_type == \"regression\":\n            # AdaBoost.R2 weight update\n            errors = np.abs(preds - pseudo_targets)  # absolute error of residual fit\n            if errors.max() > 0:\n                errors = errors / errors.max()  # normalize to [0,1]\n            avg_loss = np.dot(sample_weights, errors)\n            avg_loss = np.clip(avg_loss, 1e-10, 1.0 - 1e-10)\n            beta = avg_loss / (1.0 - avg_loss)\n            # Decrease weight for well-predicted samples (small errors)\n            sample_weights = sample_weights * np.power(beta, 1.0 - errors)\n        else:\n            # classification: confidence-weighted exponential update\n            # Compute temperature decay\n            temperature = self.temperature_base - (self.temperature_base - 1.0) * (round_idx / self.n_rounds)\n            # margin using raw scores stored in compute_targets\n            y_signed = 2 * y - 1\n            margin = y_signed * self._raw_scores\n            # gradient magnitude = |pseudo_targets|\n            grad_mag = np.abs(pseudo_targets)\n            # confidence score: high grad_mag + low margin => high weight\n            score = grad_mag * np.exp(-margin)\n            if score.max() > 0:\n                score = score / score.max()\n            # exponential weighting\n            weight_update = np.exp(temperature * score)\n            sample_weights = sample_weights * weight_update\n\n        # Normalize with clipping\n        sample_weights = np.clip(sample_weights, 1e-10, None)\n        sample_weights = sample_weights / sample_weights.sum()\n        return sample_weights"
7}

1{
2  "n": 1
3}