1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_imputation.py",
4  "start_line": 41,
5  "end_line": 131,
6  "content": "class CustomImputer(BaseEstimator, TransformerMixin):\n    \"\"\"Correlation-Enhanced Iterative Forest Imputation (CEIFI).\n\n    Algorithm:\n    1. Compute pairwise feature correlations from observed data.\n    2. For each feature, select top-k most correlated features as predictors.\n    3. Initialize missing values with column means.\n    4. Iterate up to max_iter:\n        a. For each feature with missing values (sorted by missingness):\n            - Use selected predictors to train a RandomForest regressor.\n            - Predict missing values.\n        b. Check convergence (normalized difference < tol).\n    5. Return imputed matrix.\n\n    This approach leverages feature correlations to reduce noise and improve\n    imputation accuracy, especially when features have varying relevance.\n    \"\"\"\n\n    def __init__(self, random_state=42, max_iter=10):\n        self.random_state = random_state\n        self.max_iter = max_iter\n        self.n_estimators = 100\n        self.top_k = 5  # number of correlated features to use\n        self.tol = 1e-4\n\n    def fit(self, X, y=None):\n        # Store correlation matrix and predictor indices\n        self._compute_correlations(X)\n        # Fit by running imputation on X (store state)\n        self._X_fitted = X.copy()\n        self._fit_transform_internal(X)\n        return self\n\n    def transform(self, X):\n        return self._fit_transform_internal(X)\n\n    def fit_transform(self, X, y=None):\n        return self._fit_transform_internal(X)\n\n    def _compute_correlations(self, X):\n        \"\"\"Compute pairwise correlations and select top-k predictors per feature.\"\"\"\n        n_features = X.shape[1]\n        # Use helper function to compute correlations\n        corr = compute_feature_correlations(X)\n        self.corr_matrix_ = corr\n        self.predictor_indices_ = []\n        for j in range(n_features):\n            # Get absolute correlations, exclude self\n            abs_corr = np.abs(corr[j])\n            abs_corr[j] = -1  # ignore self\n            # Top k indices (largest absolute correlation)\n            top_indices = np.argsort(abs_corr)[-self.top_k:]\n            self.predictor_indices_.append(top_indices)\n\n    def _fit_transform_internal(self, X):\n        from sklearn.ensemble import RandomForestRegressor\n\n        X_imp = X.copy()\n        n_samples, n_features = X_imp.shape\n\n        # Step 1: Initial imputation with column means\n        col_means = np.nanmean(X_imp, axis=0)\n        for j in range(n_features):\n            mask_j = np.isnan(X_imp[:, j])\n            X_imp[mask_j, j] = col_means[j]\n\n        # If correlations not computed yet, compute them\n        if not hasattr(self, 'predictor_indices_'):\n            self._compute_correlations(X_imp)\n\n        # Identify which features have missing values and sort by missingness\n        miss_count = np.isnan(X).sum(axis=0)\n        features_with_missing = np.where(miss_count > 0)[0]\n        # Sort by number of missing values (ascending)\n        features_with_missing = features_with_missing[\n            np.argsort(miss_count[features_with_missing])\n        ]\n\n        if len(features_with_missing) == 0:\n            return X_imp\n\n        # Step 2: Iterative imputation\n        for iteration in range(self.max_iter):\n            X_prev = X_imp.copy()\n\n            for j in features_with_missing:\n                # Observed and missing indices for feature j\n                obs_mask = ~np.isnan(X[:, j])\n                mis_mask = np.isnan(X[:, j])\n\n                if mis_mask.sum() == 0:\n                    continue\n\n                # Predictor features (top correlated ones)\n                pred_indices = self.predictor_indices_[j]\n                # Ensure predictors do not include j (should be already excluded)\n                pred_indices = [k for k in pred_indices if k != j]\n                # If no predictors left, use all other features\n                if len(pred_indices) == 0:\n                    pred_indices = [k for k in range(n_features) if k != j]\n\n                X_train = X_imp[obs_mask][:, pred_indices]\n                y_train = X[obs_mask, j]  # Use original observed values\n                X_pred = X_imp[mis_mask][:, pred_indices]\n\n                # Train random forest and predict\n                rf = RandomForestRegressor(\n                    n_estimators=self.n_estimators,\n                    max_features=\"sqrt\",\n                    random_state=self.random_state,\n                    n_jobs=-1,\n                )\n                rf.fit(X_train, y_train)\n                X_imp[mis_mask, j] = rf.predict(X_pred)\n\n            # Step 3: Check convergence\n            diff = np.sum((X_imp - X_prev) ** 2)\n            denom = np.sum(X_imp ** 2)\n            if denom > 0 and diff / denom < self.tol:\n                break\n\n        return X_imp\n\n\ndef compute_feature_correlations(X):\n    \"\"\"Compute pairwise correlations, ignoring NaN pairs.\n\n    Args:\n        X: array of shape (n_samples, n_features) with possible NaN values\n\n    Returns:\n        corr: array of shape (n_features, n_features) with correlation coefficients\n    \"\"\"\n    n_features = X.shape[1]\n    corr = np.eye(n_features)\n    for i in range(n_features):\n        for j in range(i + 1, n_features):\n            mask = ~(np.isnan(X[:, i]) | np.isnan(X[:, j]))\n            if mask.sum() > 2:\n                c = np.corrcoef(X[mask, i], X[mask, j])[0, 1]\n                corr[i, j] = corr[j, i] = c if not np.isnan(c) else 0.0\n    return corr"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_imputation.py",
4  "start_line": 41,
5  "end_line": 182,
6  "content": "class CustomImputer(BaseEstimator, TransformerMixin):\n    \"\"\"SoftImpute: Matrix completion via nuclear norm regularization.\n\n    Implements the SoftImpute algorithm (Mazumder et al., 2010):\n    1. Initialize missing entries with column means.\n    2. Repeat until convergence:\n        a. Compute SVD of current completed matrix.\n        b. Apply soft-thresholding to singular values: shrink(s, lambda) = max(s - lambda, 0).\n        c. Reconstruct low-rank matrix.\n        d. Replace missing entries with reconstructed values; keep observed entries fixed.\n        e. Check convergence (relative change in imputed values).\n\n    The nuclear norm regularization encourages low-rank solutions, which is appropriate\n    when features are correlated and the data matrix is approximately low-rank.\n    \"\"\"\n\n    def __init__(self, random_state=42, max_iter=10):\n        self.random_state = random_state\n        self.max_iter = max_iter\n        self.lambda_ = None  # will be set based on data\n        self.tol = 1e-4\n\n    def fit(self, X, y=None):\n        # Fit by running imputation on X (store state)\n        self._X_fitted = X.copy()\n        self._fit_transform_internal(X)\n        return self\n\n    def transform(self, X):\n        return self._fit_transform_internal(X)\n\n    def fit_transform(self, X, y=None):\n        return self._fit_transform_internal(X)\n\n    def _fit_transform_internal(self, X):\n        X_imp = X.copy()\n        n_samples, n_features = X_imp.shape\n\n        # Step 1: Initial imputation with column means\n        col_means = np.nanmean(X_imp, axis=0)\n        for j in range(n_features):\n            mask_j = np.isnan(X_imp[:, j])\n            X_imp[mask_j, j] = col_means[j]\n\n        # Determine lambda (regularization parameter) if not set\n        if self.lambda_ is None:\n            # Heuristic: lambda = 0.5 * sqrt(n_samples * n_features) * sigma_est\n            # Estimate noise variance via column variances\n            col_vars = np.nanvar(X, axis=0)\n            sigma_est = np.sqrt(np.nanmean(col_vars))\n            self.lambda_ = 0.5 * np.sqrt(max(n_samples, n_features)) * sigma_est\n\n        # Identify missing positions (fixed across iterations)\n        missing_mask = np.isnan(X)\n        observed_mask = ~missing_mask\n\n        # Step 2: Iterative soft-thresholded SVD\n        for iteration in range(self.max_iter):\n            X_prev = X_imp.copy()\n\n            # Compute SVD of current completed matrix\n            # Use randomized SVD for efficiency\n            from sklearn.utils.extmath import randomized_svd\n            U, s, Vt = randomized_svd(X_imp, n_components=min(n_samples, n_features, 50),\n                                      random_state=self.random_state)\n\n            # Soft-threshold singular values\n            s_thresh = np.maximum(s - self.lambda_, 0.0)\n\n            # Reconstruct low-rank matrix\n            X_lr = U @ np.diag(s_thresh) @ Vt\n\n            # Update missing entries with reconstructed values\n            X_imp[missing_mask] = X_lr[missing_mask]\n\n            # Keep observed entries unchanged (they are already unchanged because we didn't modify them)\n\n            # Check convergence\n            diff = np.linalg.norm(X_imp[missing_mask] - X_prev[missing_mask])\n            norm = np.linalg.norm(X_imp[missing_mask])\n            if norm > 0 and diff / norm < self.tol:\n                break\n\n        return X_imp\n\n\ndef compute_feature_correlations(X):\n    \"\"\"Compute pairwise correlations, ignoring NaN pairs.\n\n    Args:\n        X: array of shape (n_samples, n_features) with possible NaN values\n\n    Returns:\n        corr: array of shape (n_features, n_features) with correlation coefficients\n    \"\"\"\n    n_features = X.shape[1]\n    corr = np.eye(n_features)\n    for i in range(n_features):\n        for j in range(i + 1, n_features):\n            mask = ~(np.isnan(X[:, i]) | np.isnan(X[:, j]))\n            if mask.sum() > 2:\n                c = np.corrcoef(X[mask, i], X[mask, j])[0, 1]\n                corr[i, j] = corr[j, i] = c if not np.isnan(c) else 0.0\n    return corr"
7}

1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_imputation.py",
4  "start_line": 41,
5  "end_line": 144,
6  "content": "class CustomImputer(BaseEstimator, TransformerMixin):\n    \"\"\"Adaptive Iterative Imputation with Ensemble of Regressors (AIER).\n\n    Algorithm:\n    1. Initialize missing values with column means.\n    2. For each iteration until convergence:\n        a. For each feature with missing values (sorted by missingness):\n            - Use all other features as predictors.\n            - Train three base regressors: BayesianRidge, RandomForest, KNN.\n            - Compute cross-validation errors on observed data for each regressor.\n            - Set weights inversely proportional to errors.\n            - Train each regressor on all observed data and predict missing values.\n            - Combine predictions using weighted average.\n        b. Update missing entries.\n        c. Check convergence.\n\n    This adaptively blends linear, tree-based, and neighbor-based models,\n    leveraging their complementary strengths for different feature relationships.\n    \"\"\"\n\n    def __init__(self, random_state=42, max_iter=10):\n        self.random_state = random_state\n        self.max_iter = max_iter\n        self.n_estimators = 100\n        self.n_neighbors = 5\n        self.tol = 1e-4\n\n    def fit(self, X, y=None):\n        # Store the fitted state by running fit_transform internally\n        self._X_fitted = X.copy()\n        self._fit_transform_internal(X)\n        return self\n\n    def transform(self, X):\n        return self._fit_transform_internal(X)\n\n    def fit_transform(self, X, y=None):\n        return self._fit_transform_internal(X)\n\n    def _fit_transform_internal(self, X):\n        from sklearn.linear_model import BayesianRidge\n        from sklearn.ensemble import RandomForestRegressor\n        from sklearn.neighbors import KNeighborsRegressor\n        from sklearn.model_selection import KFold\n\n        X_imp = X.copy()\n        n_samples, n_features = X_imp.shape\n\n        # Step 1: Initial imputation with column means\n        col_means = np.nanmean(X_imp, axis=0)\n        for j in range(n_features):\n            mask_j = np.isnan(X_imp[:, j])\n            X_imp[mask_j, j] = col_means[j]\n\n        # Identify which features have missing values and sort by missingness\n        miss_count = np.isnan(X).sum(axis=0)\n        features_with_missing = np.where(miss_count > 0)[0]\n        # Sort by number of missing values (ascending)\n        features_with_missing = features_with_missing[\n            np.argsort(miss_count[features_with_missing])\n        ]\n\n        if len(features_with_missing) == 0:\n            return X_imp\n\n        # Prepare base regressors\n        br = BayesianRidge()\n        rf = RandomForestRegressor(\n            n_estimators=self.n_estimators,\n            max_features=\"sqrt\",\n            random_state=self.random_state,\n            n_jobs=-1\n        )\n        knn = KNeighborsRegressor(\n            n_neighbors=self.n_neighbors,\n            weights='distance'\n        )\n        regressors = [br, rf, knn]\n        n_reg = len(regressors)\n\n        # Step 2: Iterative imputation\n        for iteration in range(self.max_iter):\n            X_prev = X_imp.copy()\n\n            for j in features_with_missing:\n                obs_mask = ~np.isnan(X[:, j])\n                mis_mask = np.isnan(X[:, j])\n                if mis_mask.sum() == 0:\n                    continue\n\n                # Predictor features (all except j)\n                other_features = [k for k in range(n_features) if k != j]\n                X_all = X_imp[:, other_features]\n                y_true = X[:, j]  # original with NaNs\n\n                # Observed data\n                X_obs = X_all[obs_mask]\n                y_obs = y_true[obs_mask]\n\n                # If too few observations, fall back to mean\n                if len(y_obs) < 5:\n                    X_imp[mis_mask, j] = col_means[j]\n                    continue\n\n                # Compute cross-validation errors for weighting\n                cv = KFold(n_splits=min(5, len(y_obs)), shuffle=True, random_state=self.random_state)\n                errors = []\n                for reg in regressors:\n                    preds = []\n                    truths = []\n                    for train_idx, val_idx in cv.split(X_obs):\n                        reg_clone = self._clone_regressor(reg)\n                        reg_clone.fit(X_obs[train_idx], y_obs[train_idx])\n                        pred = reg_clone.predict(X_obs[val_idx])\n                        preds.extend(pred)\n                        truths.extend(y_obs[val_idx])\n                    mse = np.mean((np.array(preds) - np.array(truths)) ** 2)\n                    errors.append(mse + 1e-8)  # avoid zero\n\n                # Weights inversely proportional to errors\n                inv_errors = 1.0 / np.array(errors)\n                weights = inv_errors / inv_errors.sum()\n\n                # Train each regressor on full observed data and predict missing values\n                predictions = np.zeros((mis_mask.sum(), n_reg))\n                for idx, reg in enumerate(regressors):\n                    reg_clone = self._clone_regressor(reg)\n                    reg_clone.fit(X_obs, y_obs)\n                    X_mis = X_all[mis_mask]\n                    predictions[:, idx] = reg_clone.predict(X_mis)\n\n                # Weighted average\n                weighted_pred = np.dot(predictions, weights)\n                X_imp[mis_mask, j] = weighted_pred\n\n            # Check convergence\n            diff = np.sum((X_imp - X_prev) ** 2)\n            denom = np.sum(X_imp ** 2)\n            if denom > 0 and diff / denom < self.tol:\n                break\n\n        return X_imp\n\n    def _clone_regressor(self, reg):\n        \"\"\"Return a fresh copy of the regressor.\"\"\"\n        from sklearn.base import clone\n        return clone(reg)\n\n\ndef compute_feature_correlations(X):\n    \"\"\"Compute pairwise correlations, ignoring NaN pairs.\n\n    Args:\n        X: array of shape (n_samples, n_features) with possible NaN values\n\n    Returns:\n        corr: array of shape (n_features, n_features) with correlation coefficients\n    \"\"\"\n    n_features = X.shape[1]\n    corr = np.eye(n_features)\n    for i in range(n_features):\n        for j in range(i + 1, n_features):\n            mask = ~(np.isnan(X[:, i]) | np.isnan(X[:, j]))\n            if mask.sum() > 2:\n                c = np.corrcoef(X[mask, i], X[mask, j])[0, 1]\n                corr[i, j] = corr[j, i] = c if not np.isnan(c) else 0.0\n    return corr"
7}

1{
2  "n": 3
3}