1{
2  "op": "replace",
3  "filename": "scikit-learn/custom_anomaly.py",
4  "start_line": 160,
5  "end_line": 212,
6  "content": "class CustomAnomalyDetector:\n    \"\"\"Multi-View Adaptive Fusion Anomaly Detector (MVAF).\n\n    Novel algorithm combining multiple detection paradigms with\n    correlation-aware adaptive fusion. Key innovations:\n    1. Multi-view scoring: ECOD (marginal), IForest (structural),\n       COPOD (dependency), PCA reconstruction (subspace)\n    2. Correlation-aware fusion: Down-weights correlated/\n       redundant detectors using inverse pairwise correlation\n    3. Multi-scale subsampling: Robust scores from multiple\n       subsample sizes to capture local and global anomalies\n    4. Stability-weighted aggregation: Weights by score variance\n       across subsamples\n    \"\"\"\n\n    def __init__(self):\n        self.n_subscales = 3\n        self.subsample_ratios = [0.5, 0.7, 1.0]\n        self.random_state = 42\n\n    def fit(self, X):\n        \"\"\"Fit detector on unlabeled training data.\"\"\"\n        X = np.asarray(X, dtype=np.float64)\n        self.n_train = X.shape[0]\n        self.n_features = X.shape[1]\n\n        # Store training data for ECOD\n        self._sorted_train = np.sort(X, axis=0)\n\n        # Compute skewness for ECOD\n        from scipy.stats import skew\n        self._skewness_sign = np.sign(skew(X, axis=0))\n\n        # Fit PCA for reconstruction error\n        from sklearn.decomposition import PCA\n        # Adaptive n_components based on data dimensionality\n        n_comp = min(max(int(self.n_features * 0.7), 3), self.n_features - 1)\n        n_comp = min(n_comp, self.n_train - 1)\n        self.pca = PCA(n_components=n_comp, random_state=self.random_state)\n        self.pca.fit(X)\n\n        # Fit IForest\n        from pyod.models.iforest import IForest\n        n_estimators = min(200, max(100, self.n_train // 10))\n        self.iforest = IForest(\n            n_estimators=n_estimators,\n            max_samples=min(256, max(64, self.n_train // 4)),\n            random_state=self.random_state,\n        )\n        self.iforest.fit(X)\n\n        # Fit COPOD\n        from pyod.models.copod import COPOD\n        self.copod = COPOD()\n        self.copod.fit(X)\n\n        # Pre-compute PCA reconstruction error on training data\n        X_pca = self.pca.transform(X)\n        X_reconstructed = self.pca.inverse_transform(X_pca)\n        self.train_recon_error = np.sum((X - X_reconstructed) ** 2, axis=1)\n\n        # Generate multi-scale scores on training data for correlation analysis\n        self._fit_ensemble_weights(X)\n\n        return self\n\n    def _ecod_scores(self, X):\n        \"\"\"Compute ECOD scores.\"\"\"\n        X = np.asarray(X, dtype=np.float64)\n        n = self._sorted_train.shape[0]\n        d = X.shape[1]\n        scores = np.zeros(X.shape[0])\n        for j in range(d):\n            ranks = np.searchsorted(self._sorted_train[:, j], X[:, j], side='right')\n            ecdf_left = ranks / (n + 1)\n            ecdf_right = 1.0 - ecdf_left\n            ecdf_left = np.clip(ecdf_left, 1e-10, 1.0)\n            ecdf_right = np.clip(ecdf_right, 1e-10, 1.0)\n            u_left = -np.log(ecdf_left)\n            u_right = -np.log(ecdf_right)\n            u_j = np.maximum(u_left, u_right)\n            scores += u_j\n        return scores\n\n    def _recon_scores(self, X):\n        \"\"\"Compute PCA reconstruction error scores.\"\"\"\n        X = np.asarray(X, dtype=np.float64)\n        X_pca = self.pca.transform(X)\n        X_reconstructed = self.pca.inverse_transform(X_pca)\n        return np.sum((X - X_reconstructed) ** 2, axis=1)\n\n    def _compute_detectors(self, X_train, X_test):\n        \"\"\"Compute all detector scores for correlation analysis.\"\"\"\n        # ECOD\n        ecod_train = self._ecod_scores(X_train)\n        ecod_test = self._ecod_scores(X_test)\n\n        # IForest\n        iforest_train = self.iforest.decision_function(X_train)\n        iforest_test = self.iforest.decision_function(X_test)\n\n        # COPOD\n        copod_train = self.copod.decision_function(X_train)\n        copod_test = self.copod.decision_function(X_test)\n\n        # PCA Reconstruction\n        recon_train = self._recon_scores(X_train)\n        recon_test = self._recon_scores(X_test)\n\n        train_scores = np.column_stack([ecod_train, iforest_train, copod_train, recon_train])\n        test_scores = np.column_stack([ecod_test, iforest_test, copod_test, recon_test])\n\n        return train_scores, test_scores\n\n    def _fit_ensemble_weights(self, X):\n        \"\"\"Compute adaptive weights based on detector diversity and stability.\"\"\"\n        rng = np.random.RandomState(self.random_state)\n\n        n_detectors = 4\n        n_subsamples = len(self.subsample_ratios)\n\n        # Collect scores across multiple subsamples for stability analysis\n        all_train_scores = []\n        test_indices = rng.choice(self.n_train, size=min(500, self.n_train // 5), replace=False)\n        X_test_sub = X[test_indices]\n\n        for ratio in self.subsample_ratios:\n            n_sub = max(int(self.n_train * ratio), 100)\n            sub_indices = rng.choice(self.n_train, size=n_sub, replace=False)\n            X_sub = X[sub_indices]\n\n            # Recompute detector scores on subsample\n            # For IForest and COPOD, fit on subsample\n            from pyod.models.iforest import IForest\n            from pyod.models.copod import COPOD\n\n            if_sub = IForest(n_estimators=min(100, n_sub // 5), random_state=rng.randint(0, 10000))\n            if_sub.fit(X_sub)\n\n            co_sub = COPOD()\n            co_sub.fit(X_sub)\n\n            # ECOD on subsample\n            sorted_sub = np.sort(X_sub, axis=0)\n            n_s = X_sub.shape[0]\n            d = X_sub.shape[1]\n            ecod_test = np.zeros(X_test_sub.shape[0])\n            for j in range(d):\n                ranks = np.searchsorted(sorted_sub[:, j], X_test_sub[:, j], side='right')\n                ecdf_left = ranks / (n_s + 1)\n                ecdf_right = 1.0 - ecdf_left\n                ecdf_left = np.clip(ecdf_left, 1e-10, 1.0)\n                ecdf_right = np.clip(ecdf_right, 1e-10, 1.0)\n                u_j = np.maximum(-np.log(ecdf_left), -np.log(ecdf_right))\n                ecod_test += u_j\n\n            # Reconstruction (using same PCA)\n            X_pca = self.pca.transform(X_test_sub)\n            X_rec = self.pca.inverse_transform(X_pca)\n            recon_test = np.sum((X_test_sub - X_rec) ** 2, axis=1)\n\n            # Standardize all scores\n            scores = np.column_stack([\n                ecod_test,\n                if_sub.decision_function(X_test_sub),\n                co_sub.decision_function(X_test_sub),\n                recon_test,\n            ])\n            # Row-standardize each detector\n            for i in range(n_detectors):\n                col = scores[:, i]\n                std = col.std()\n                if std > 1e-10:\n                    scores[:, i] = (col - col.mean()) / std\n                else:\n                    scores[:, i] = 0.0\n            all_train_scores.append(scores)\n\n        all_train_scores = np.array(all_train_scores)  # (n_scales, n_points, n_detectors)\n\n        # Compute stability weight: inverse of variance across subscales\n        # Lower variance = more stable = higher weight\n        variances = all_train_scores.var(axis=0)  # (n_points, n_detectors)\n        stability = 1.0 / (variances.mean(axis=0) + 1e-6)  # (n_detectors,)\n        stability = stability / stability.sum()\n\n        # Compute diversity weight from full model\n        train_scores, _ = self._compute_detectors(X, X[test_indices])\n        # Compute pairwise absolute correlation\n        corr_matrix = np.corrcoef(train_scores.T)\n        np.fill_diagonal(corr_matrix, 0)\n        abs_corr = np.abs(corr_matrix)\n        # Diversity: lower correlation with others = higher weight\n        diversity = 1.0 / (abs_corr.mean(axis=1) + 1e-6)\n        diversity = diversity / diversity.sum()\n\n        # Combine stability and diversity\n        self.ensemble_weights = 0.5 * stability + 0.5 * diversity\n        self.ensemble_weights = self.ensemble_weights / self.ensemble_weights.sum()\n\n        # Normalize training reconstruction errors for scoring\n        self.train_recon_mean = self.train_recon_error.mean()\n        self.train_recon_std = self.train_recon_error.std()\n        if self.train_recon_std < 1e-10:\n            self.train_recon_std = 1.0\n\n    def decision_function(self, X):\n        \"\"\"Compute anomaly scores for input data.\"\"\"\n        X = np.asarray(X, dtype=np.float64)\n\n        # ECOD scores\n        ecod = self._ecod_scores(X)\n\n        # IForest scores\n        iforest = self.iforest.decision_function(X)\n\n        # COPOD scores\n        copod = self.copod.decision_function(X)\n\n        # PCA reconstruction scores (standardized)\n        recon = self._recon_scores(X)\n        recon = (recon - self.train_recon_mean) / self.train_recon_std\n\n        # Combine into matrix (n_samples, 4)\n        scores_matrix = np.column_stack([ecod, iforest, copod, recon])\n\n        # Standardize each column using training statistics\n        for i in range(4):\n            col = scores_matrix[:, i]\n            col_std = col.std()\n            if col_std > 1e-10:\n                scores_matrix[:, i] = (col - col.mean()) / col_std\n            else:\n                scores_matrix[:, i] = 0.0\n\n        # Weighted fusion\n        final_scores = scores_matrix @ self.ensemble_weights\n\n        return final_scores"
7}

1{
2  "op": "replace",
3  "content": "class CustomAnomalyDetector:\n    \"\"\"Cross-Correlation Weighted Rank Ensemble (CCWRE).\n\n    Novel unsupervised anomaly detection algorithm with three contributions:\n\n    1. Rank-Based Cross-Detector Combination: Convert raw scores from each\n       detector to ranks, then combine using rank averaging. This is robust to\n       different score distributions across detectors and datasets.\n\n    2. Correlation-Weighted Fusion: Assign diversity-aware weights to each\n       detector inversely proportional to its average pairwise correlation\n       with others. Less correlated detectors contribute more, capturing\n       complementary anomaly signals.\n\n    3. Multi-Subspace LODA Integration: Include sub-space anomaly detection\n       via LODA with multiple random projections to capture anomalies\n       manifesting only in feature subspaces.\n\n    Detectors used: ECOD (marginal), IForest (structural), COPOD (dependency),\n    LODA (subspace). All scores converted to ranks, then diversity-weighted\n    combined.\n    \"\"\"\n\n    def __init__(self):\n        self.random_state = 42\n\n    def fit(self, X):\n        \"\"\"Fit detector on unlabeled training data.\"\"\"\n        import numpy as np\n        from scipy.stats import skew\n        from sklearn.decomposition import PCA\n\n        np.random.seed(self.random_state)\n        X = np.asarray(X, dtype=np.float64)\n        self.n_train = X.shape[0]\n        self.n_features = X.shape[1]\n\n        # --- Store data for inductive ECOD scoring ---\n        self._sorted_train = np.sort(X, axis=0)\n        self._skewness_sign = np.sign(skew(X, axis=0))\n\n        # --- Fit IForest ---\n        from pyod.models.iforest import IForest\n        n_est = min(200, max(100, self.n_train // 10))\n        ms = min(256, max(64, self.n_train // 4))\n        self.iforest = IForest(\n            n_estimators=n_est,\n            max_samples=ms,\n            random_state=self.random_state,\n        )\n        self.iforest.fit(X)\n\n        # --- Fit COPOD ---\n        from pyod.models.copod import COPOD\n        self.copod = COPOD()\n        self.copod.fit(X)\n\n        # --- Fit multiple LODAs with different random projections ---\n        from pyod.models.loda import LODA\n        self.loda_models = []\n        n_loda = 3\n        for i in range(n_loda):\n            loda = LODA(\n                n_bins=10,\n                n_random_cuts=8,\n                random_state=self.random_state + i,\n            )\n            loda.fit(X)\n            self.loda_models.append(loda)\n\n        # --- Compute correlation weights on training data ---\n        self._compute_weights(X)\n\n        return self\n\n    def _ecod_scores(self, X):\n        \"\"\"Compute inductive ECOD scores (no data leakage).\"\"\"\n        import numpy as np\n        X = np.asarray(X, dtype=np.float64)\n        n = self._sorted_train.shape[0]\n        d = X.shape[1]\n        scores = np.zeros(X.shape[0])\n        for j in range(d):\n            ranks = np.searchsorted(self._sorted_train[:, j], X[:, j], side='right')\n            ecdf_left = ranks / (n + 1)\n            ecdf_right = 1.0 - ecdf_left\n            ecdf_left = np.clip(ecdf_left, 1e-10, 1.0)\n            ecdf_right = np.clip(ecdf_right, 1e-10, 1.0)\n            u_j = np.maximum(-np.log(ecdf_left), -np.log(ecdf_right))\n            scores += u_j\n        return scores\n\n    def _to_ranks(self, scores):\n        \"\"\"Convert scores to rank-based scores (higher rank = more anomalous).\"\"\"\n        import numpy as np\n        n = len(scores)\n        # argsort twice gives ranks\n        order = np.argsort(scores, kind='mergesort')\n        ranks = np.empty(n, dtype=np.float64)\n        ranks[order] = np.arange(n, dtype=np.float64)\n        return ranks / (n - 1) if n > 1 else ranks\n\n    def _compute_weights(self, X):\n        \"\"\"Compute diversity-aware ensemble weights.\"\"\"\n        import numpy as np\n\n        # Get all detector scores on training data\n        ecod = self._ecod_scores(X)\n        iforest = self.iforest.decision_function(X)\n        copod = self.copod.decision_function(X)\n        loda_scores = [m.decision_function(X) for m in self.loda_models]\n\n        all_scores = [ecod, iforest, copod] + loda_scores\n        n_detectors = len(all_scores)\n\n        # Convert to ranks for correlation computation (more robust)\n        rank_matrix = np.column_stack([\n            self._to_ranks(s) for s in all_scores\n        ])\n\n        # Compute pairwise Pearson correlation\n        corr_matrix = np.corrcoef(rank_matrix.T)\n        np.fill_diagonal(corr_matrix, 0)\n\n        # Diversity weight: 1 / (1 + avg_abs_corr)\n        abs_corr = np.abs(corr_matrix)\n        avg_abs_corr = abs_corr.mean(axis=1)\n        diversity = 1.0 / (1.0 + avg_abs_corr)\n        diversity = diversity / diversity.sum()\n\n        self.weights = diversity\n\n        # Store individual scores on training data for reference\n        # (not used in predict, but stored for potential calibration)\n        self.train_score_means = np.array([s.mean() for s in all_scores])\n        self.train_score_stds = np.array([s.std() + 1e-10 for s in all_scores])\n\n    def decision_function(self, X):\n        \"\"\"Compute anomaly scores for input data.\"\"\"\n        import numpy as np\n        X = np.asarray(X, dtype=np.float64)\n\n        # --- Collect raw scores from all detectors ---\n        ecod = self._ecod_scores(X)\n        iforest = self.iforest.decision_function(X)\n        copod = self.copod.decision_function(X)\n        loda_scores = [m.decision_function(X) for m in self.loda_models]\n\n        all_scores = [ecod, iforest, copod] + loda_scores\n\n        # --- Convert each to ranks ---\n        rank_matrix = np.column_stack([\n            self._to_ranks(s) for s in all_scores\n        ])\n\n        # --- Weighted rank combination ---\n        final_scores = rank_matrix @ self.weights\n\n        # --- Amplify tail separation via soft-thresholding ---\n        # Shift so median is at 0, then apply exp for tail emphasis\n        median_score = np.median(final_scores)\n        shifted = final_scores - median_score\n        # Soft amplification: preserves ordering but spreads tail\n        amplified = shifted * (1 + np.abs(shifted) * 0.5)\n\n        return amplified",
4  "filename": "scikit-learn/custom_anomaly.py",
5  "start_line": 160,
6  "end_line": 398
7}

1{
2  "op": "create",
3  "content": "# Check available data\nimport os\nimport glob\n\npaths_checked = []\n\n# Check various locations\nfor base in [\"/data\", \"/data/adbench\", \"/srv\", \"/tmp\", \".\"]:\n    if os.path.exists(base):\n        paths_checked.append(base)\n        print(f\"\\n{base}: exists\")\n        try:\n            entries = os.listdir(base)\n            print(f\"  Contents ({len(entries)} items): {entries[:20]}\")\n        except:\n            pass\n\n# Search for npz files\nprint(\"\\nSearching for .npz files...\")\nfor root, dirs, files in os.walk(\"/data\", topdown=3):\n    for f in files:\n        if f.endswith(\".npz\"):\n            print(os.path.join(root, f))\n    if len(dirs) > 20:\n        dirs[:] = dirs[:5]  # limit depth\n\nfor root, dirs, files in os.walk(\"/srv\", topdown=3):\n    for f in files:\n        if f.endswith(\".npz\"):\n            print(os.path.join(root, f))\n    if len(dirs) > 20:\n        dirs[:] = dirs[:5]\n",
4  "filename": "scikit-learn/check_data.py"
5}

1{
2  "op": "replace",
3  "content": "DATA_DIR = os.environ.get(\"AD_DATA_DIR\", \"/data/adbench\")\nif not os.path.isdir(DATA_DIR):\n    # Search for the data\n    for candidate in [\n        os.path.expanduser(\"~/adbench\"),\n        os.path.expanduser(\"~/.kaggle/datasets\"),\n        \"/tmp/adbench\",\n        \"./adbench\",\n    ]:\n        if os.path.isdir(candidate):\n            DATA_DIR = candidate\n            break",
4  "filename": "scikit-learn/custom_anomaly.py",
5  "start_line": 32,
6  "end_line": 32
7}

1{
2  "n": 1
3}

1{
2  "n": 2
3}

1{
2  "content": "class CustomAnomalyDetector:\n    \"\"\"Cross-Correlation Weighted Rank Ensemble (CCWRE).\n\n    Novel unsupervised anomaly detection algorithm combining multiple\n    orthogonal detection paradigms with diversity-aware fusion.\n\n    Contributions:\n    1. Multi-paradigm ensemble: ECOD (marginal tails), IForest (tree isolation),\n       COPOD (copula dependencies), LODA (random projection subspaces).\n    2. Diversity-weighted rank fusion: Converts raw scores to ranks,\n       weights each detector inversely proportional to its average pairwise\n       correlation with others, ensuring complementary signals are amplified.\n    3. Multi-instance LODA: Multiple LODA models with different random seeds\n       capture anomalies in different random subspaces.\n    4. Tail-amplified scoring: Median-centered soft amplification spreads\n       tail scores for better threshold separation.\n    \"\"\"\n\n    def __init__(self):\n        self.random_state = 42\n\n    def fit(self, X):\n        \"\"\"Fit detector on unlabeled training data.\"\"\"\n        import numpy as np\n        from scipy.stats import skew\n\n        np.random.seed(self.random_state)\n        X = np.asarray(X, dtype=np.float64)\n        self.n_train = X.shape[0]\n        self.n_features = X.shape[1]\n\n        # Store sorted data for inductive ECOD scoring\n        self._sorted_train = np.sort(X, axis=0)\n\n        # Fit IForest\n        from pyod.models.iforest import IForest\n        n_est = min(200, max(100, self.n_train // 10))\n        ms = min(256, max(64, self.n_train // 4))\n        self.iforest = IForest(\n            n_estimators=n_est,\n            max_samples=ms,\n            random_state=self.random_state,\n        )\n        self.iforest.fit(X)\n\n        # Fit COPOD\n        from pyod.models.copod import COPOD\n        self.copod = COPOD()\n        self.copod.fit(X)\n\n        # Fit multiple LODA models with different random projections\n        from pyod.models.loda import LODA\n        self.loda_models = []\n        n_loda = 3\n        for i in range(n_loda):\n            loda = LODA(\n                n_bins=10,\n                n_random_cuts=8,\n                random_state=self.random_state + i + 1,\n            )\n            loda.fit(X)\n            self.loda_models.append(loda)\n\n        # Compute diversity-aware ensemble weights\n        self._compute_weights(X)\n\n        # Store reference stats for scoring\n        ecod_ref = self._ecod_scores(X)\n        iforest_ref = self.iforest.decision_function(X)\n        copod_ref = self.copod.decision_function(X)\n        loda_refs = [m.decision_function(X) for m in self.loda_models]\n        all_refs = [ecod_ref, iforest_ref, copod_ref] + loda_refs\n        self.ref_means = np.array([s.mean() for s in all_refs])\n        self.ref_stds = np.array([max(s.std(), 1e-10) for s in all_refs])\n        self._fitted_median = np.median(self._combine_scores(X))\n\n        return self\n\n    def _ecod_scores(self, X):\n        \"\"\"Inductive ECOD scores (no test-data leakage).\"\"\"\n        import numpy as np\n        X = np.asarray(X, dtype=np.float64)\n        n = self._sorted_train.shape[0]\n        d = X.shape[1]\n        scores = np.zeros(X.shape[0])\n        for j in range(d):\n            ranks = np.searchsorted(self._sorted_train[:, j], X[:, j], side='right')\n            ecdf_l = np.clip(ranks / (n + 1), 1e-10, 1.0)\n            ecdf_r = np.clip(1.0 - ranks / (n + 1), 1e-10, 1.0)\n            scores += np.maximum(-np.log(ecdf_l), -np.log(ecdf_r))\n        return scores\n\n    def _to_ranks(self, scores):\n        \"\"\"Convert to normalized ranks [0, 1], higher = more anomalous.\"\"\"\n        import numpy as np\n        n = len(scores)\n        order = np.argsort(scores, kind='mergesort')\n        ranks = np.empty(n, dtype=np.float64)\n        ranks[order] = np.arange(n, dtype=np.float64)\n        return ranks / max(n - 1, 1)\n\n    def _compute_weights(self, X):\n        \"\"\"Compute correlation-based diversity weights.\"\"\"\n        import numpy as np\n        ecod = self._ecod_scores(X)\n        iforest = self.iforest.decision_function(X)\n        copod = self.copod.decision_function(X)\n        loda_scores = [m.decision_function(X) for m in self.loda_models]\n        all_scores = [ecod, iforest, copod] + loda_scores\n\n        # Rank-based correlation\n        rank_matrix = np.column_stack([self._to_ranks(s) for s in all_scores])\n        corr = np.corrcoef(rank_matrix.T)\n        np.fill_diagonal(corr, 0)\n\n        # Inverse correlation weighting\n        div = 1.0 / (1.0 + np.abs(corr).mean(axis=1))\n        self.weights = div / div.sum()\n        self.n_detectors = len(all_scores)\n\n    def _combine_scores(self, X):\n        \"\"\"Weighted rank combination of all detector scores.\"\"\"\n        import numpy as np\n        ecod = self._ecod_scores(X)\n        iforest = self.iforest.decision_function(X)\n        copod = self.copod.decision_function(X)\n        loda_scores = [m.decision_function(X) for m in self.loda_models]\n        all_scores = [ecod, iforest, copod] + loda_scores\n\n        rank_matrix = np.column_stack([self._to_ranks(s) for s in all_scores])\n        return rank_matrix @ self.weights\n\n    def decision_function(self, X):\n        \"\"\"Compute anomaly scores (higher = more anomalous).\"\"\"\n        import numpy as np\n        X = np.asarray(X, dtype=np.float64)\n\n        scores = self._combine_scores(X)\n\n        # Median-center and tail-amplify\n        shifted = scores - self._fitted_median\n        amplified = shifted * (1.0 + np.abs(shifted) * 0.5)\n        return amplified",
3  "filename": "scikit-learn/custom_anomaly.py",
4  "start_line": 160,
5  "end_line": 325
6}