From 1f778e854c16c22280151f4110c36d6464f612bc Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Thu, 18 Feb 2021 15:23:51 -0600 Subject: [PATCH] modified metadata of detection_algorithm module, FIXME: multidimensional data for MatrixProfile --- primitive_tests/test.sh | 4 +-- tods/detection_algorithm/AutoRegODetect.py | 11 ++++-- tods/detection_algorithm/DeepLog.py | 18 +++++----- tods/detection_algorithm/Ensemble.py | 16 ++++++--- tods/detection_algorithm/KDiscordODetect.py | 17 +++++---- tods/detection_algorithm/MatrixProfile.py | 27 +++++++------- tods/detection_algorithm/PCAODetect.py | 17 +++++---- tods/detection_algorithm/PyodABOD.py | 16 ++++++--- tods/detection_algorithm/PyodAE.py | 14 +++++--- tods/detection_algorithm/PyodCBLOF.py | 12 ++++--- tods/detection_algorithm/PyodCOF.py | 25 +++++-------- tods/detection_algorithm/PyodHBOS.py | 17 +++++---- tods/detection_algorithm/PyodIsolationForest.py | 13 ++++--- tods/detection_algorithm/PyodKNN.py | 13 ++++--- tods/detection_algorithm/PyodLODA.py | 15 +++++--- tods/detection_algorithm/PyodLOF.py | 15 +++++--- tods/detection_algorithm/PyodOCSVM.py | 15 +++++--- tods/detection_algorithm/PyodSOD.py | 24 +++++++------ tods/detection_algorithm/PyodSoGaal.py | 47 +++++++++---------------- tods/detection_algorithm/PyodVAE.py | 14 +++++--- tods/detection_algorithm/SystemWiseDetection.py | 47 +++++++++---------------- tods/detection_algorithm/Telemanom.py | 40 +++++++++------------ tods/feature_analysis/BKFilter.py | 28 +++++++-------- 23 files changed, 253 insertions(+), 212 deletions(-) diff --git a/primitive_tests/test.sh b/primitive_tests/test.sh index d25e921..ca073a9 100644 --- a/primitive_tests/test.sh +++ b/primitive_tests/test.sh @@ -2,13 +2,13 @@ #modules="data_processing timeseries_processing feature_analysis detection_algorithms reinforcement" #modules="data_processing timeseries_processing" -modules="feature_analysis" +modules="detection_algorithm" #test_scripts=$(ls primitive_tests | grep -v -f tested_file.txt) for module in $modules do test_scripts=$(ls $module | grep -v -f tested_file.txt) - + #test_scripts=$(ls $module) for file in $test_scripts do for f in $tested_file diff --git a/tods/detection_algorithm/AutoRegODetect.py b/tods/detection_algorithm/AutoRegODetect.py index 4a1947e..0e506f8 100644 --- a/tods/detection_algorithm/AutoRegODetect.py +++ b/tods/detection_algorithm/AutoRegODetect.py @@ -130,11 +130,16 @@ class AutoRegODetectorPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "AutoRegODetector", "python_path": "d3m.primitives.tods.detection_algorithm.AutoRegODetector", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.ISOLATION_FOREST, ], + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "version": "0.0.1", "hyperparams_to_tune": ['window_size', 'contamination', 'step_size', 'method', 'weights'], diff --git a/tods/detection_algorithm/DeepLog.py b/tods/detection_algorithm/DeepLog.py index c4b2bf2..17e115c 100644 --- a/tods/detection_algorithm/DeepLog.py +++ b/tods/detection_algorithm/DeepLog.py @@ -159,21 +159,23 @@ class DeepLogPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, """ - __author__ = "DATA Lab at Texas A&M University", - metadata = metadata_base.PrimitiveMetadata( - { + metadata = metadata_base.PrimitiveMetadata({ '__author__': "DATA Lab @Texas A&M University", 'name': "DeepLog Anomolay Detection", 'python_path': 'd3m.primitives.tods.detection_algorithm.deeplog', - 'source': {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods/-/blob/Yile/anomaly-primitives/anomaly_primitives/MatrixProfile.py']}, - 'algorithm_types': [metadata_base.PrimitiveAlgorithmType.DEEPLOG], - 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'DeepLogPrimitive')), + 'source': { + 'name': "DATALAB @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, 'hyperparams_to_tune': ['hidden_size', 'loss', 'optimizer', 'epochs', 'batch_size', 'l2_regularizer', 'validation_size', 'window_size', 'features', 'stacked_layers', 'preprocessing', 'verbose', 'dropout_rate','contamination'], 'version': '0.0.1', + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE + ], + 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'DeepLogPrimitive')), } ) diff --git a/tods/detection_algorithm/Ensemble.py b/tods/detection_algorithm/Ensemble.py index 72e80de..09ccbb3 100644 --- a/tods/detection_algorithm/Ensemble.py +++ b/tods/detection_algorithm/Ensemble.py @@ -10,6 +10,7 @@ import pandas as pd # Custom import commands if any from sklearn.preprocessing.data import Normalizer from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt +import uuid from d3m.container.numpy import ndarray as d3m_ndarray @@ -116,16 +117,21 @@ class EnsemblePrimitive(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params """ - __author__ = "DATA Lab at Texas A&M University" metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "Ensemble", - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.ISOLATION_FOREST, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "python_path": "d3m.primitives.tods.detection_algorithm.Ensemble", - "source": {'name': 'DATA Lab at Texas A&M University', 'contact': 'mailto:khlai037@tamu.edu', 'uris': ['https://gitlab.com/lhenry15/tods.git','https://gitlab.com/lhenry15/tods/-/blob/mia/anomaly-primitives/anomaly_primitives/HoltSmoothing.py']}, + "source": { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", - "id": "3688b5b4-885c-40bb-9731-fe3969ea81b0", "hyperparams_to_tune": ['use_columns'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'EnsemblePrimitive')), }) def __init__(self, *, diff --git a/tods/detection_algorithm/KDiscordODetect.py b/tods/detection_algorithm/KDiscordODetect.py index 3a6fc7c..e9789a9 100644 --- a/tods/detection_algorithm/KDiscordODetect.py +++ b/tods/detection_algorithm/KDiscordODetect.py @@ -244,15 +244,20 @@ class KDiscordODetectorPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "KDiscordODetector", "python_path": "d3m.primitives.tods.detection_algorithm.KDiscordODetector", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.LOCAL_OUTLIER_FACTOR, ], # - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + 'uris': ['https://gitlab.com/lhenry15/tods.git'] + }, "version": "0.0.1", - "hyperparams_to_tune": ['n_neighbors', 'algorithm', 'leaf_size', 'p', 'contamination', - 'window_size', 'step_size', 'method', 'radius'], + "hyperparams_to_tune": ['n_neighbors', 'algorithm', 'leaf_size', 'p', 'contamination', 'window_size', 'step_size', 'method', 'radius'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'KDiscordODetector')), }) diff --git a/tods/detection_algorithm/MatrixProfile.py b/tods/detection_algorithm/MatrixProfile.py index 23f6eb1..9b2bfa5 100644 --- a/tods/detection_algorithm/MatrixProfile.py +++ b/tods/detection_algorithm/MatrixProfile.py @@ -145,18 +145,21 @@ class MatrixProfilePrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Pa """ metadata = metadata_base.PrimitiveMetadata({ - '__author__': "DATA Lab @Texas A&M University", - 'name': "Matrix Profile", - #'python_path': 'd3m.primitives.tods.feature_analysis.matrix_profile', - 'python_path': 'd3m.primitives.tods.detection_algorithm.matrix_profile', - 'source': {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods/-/blob/Yile/anomaly-primitives/anomaly_primitives/MatrixProfile.py']}, - 'algorithm_types': [metadata_base.PrimitiveAlgorithmType.MATRIX_PROFILE,], - 'primitive_family': metadata_base.PrimitiveFamily.FEATURE_CONSTRUCTION, - 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'MatrixProfilePrimitive')), - 'hyperparams_to_tune': ['window_size'], - 'version': '0.0.2', - }) + '__author__': "DATA Lab @Texas A&M University", + 'name': "Matrix Profile", + 'python_path': 'd3m.primitives.tods.detection_algorithm.matrix_profile', + 'source': { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, + 'hyperparams_to_tune': ['window_size'], + 'version': '0.0.2', + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'MatrixProfilePrimitive')), + }) def __init__(self, *, diff --git a/tods/detection_algorithm/PCAODetect.py b/tods/detection_algorithm/PCAODetect.py index 93efdc2..13be022 100644 --- a/tods/detection_algorithm/PCAODetect.py +++ b/tods/detection_algorithm/PCAODetect.py @@ -260,16 +260,19 @@ class PCAODetectorPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Par """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "PCAODetector", "python_path": "d3m.primitives.tods.detection_algorithm.PCAODetector", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.LOCAL_OUTLIER_FACTOR, ], # - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", - "hyperparams_to_tune": ['n_components', 'n_selected_components', 'contamination', - 'whiten', 'svd_solver', 'tol', 'iterated_power', 'random_state', - 'standardization'], + "hyperparams_to_tune": ['n_components', 'n_selected_components', 'contamination', 'whiten', 'svd_solver', 'tol', 'iterated_power', 'random_state', 'standardization'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'PCAODetector')), }) diff --git a/tods/detection_algorithm/PyodABOD.py b/tods/detection_algorithm/PyodABOD.py index c77a7db..be84c71 100644 --- a/tods/detection_algorithm/PyodABOD.py +++ b/tods/detection_algorithm/PyodABOD.py @@ -22,6 +22,7 @@ from d3m import utils from d3m.base import utils as base_utils from d3m.exceptions import PrimitiveNotFittedError from d3m.primitive_interfaces.base import CallResult, DockerContainer +import uuid # from d3m.primitive_interfaces.supervised_learning import SupervisedLearnerPrimitiveBase from d3m.primitive_interfaces.unsupervised_learning import UnsupervisedLearnerPrimitiveBase @@ -116,15 +117,20 @@ class ABODPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hyp __author__: "DATA Lab at Texas A&M University" metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "Angle-base Outlier Detection Primitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_abod", - "source": {'name': 'DATA Lab at Texas A&M University', 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git', 'https://gitlab.com/lhenry15/tods/-/blob/Junjie/anomaly-primitives/anomaly_primitives/PyodABOD.py']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.ANGLE_BASE_OUTLIER_DETECTION], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "id": "134f6c5f-717b-4683-bfbc-251bab07f6fa", + "source": { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, "hyperparams_to_tune": ['contamination', 'n_neighbors', 'method'], "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'ABODPrimitive')), }) def __init__(self, *, diff --git a/tods/detection_algorithm/PyodAE.py b/tods/detection_algorithm/PyodAE.py index f34411c..784a371 100644 --- a/tods/detection_algorithm/PyodAE.py +++ b/tods/detection_algorithm/PyodAE.py @@ -271,13 +271,19 @@ class AutoEncoderPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Para """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.AutoEncoder", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_ae", - "source": {'name': "DATA Lab at Texas A&M University", 'contact': 'mailto:khlai037@tamu.edu','uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.VARIATIONAL_AUTO_ENCODER, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATA Lab at Texas A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", - "hyperparameters_to_tune": [''], + "hyperparameters_to_tune": ['contamination', 'hidden_activation, output_activation, loss'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'AutoEncoderPrimitive')), }) diff --git a/tods/detection_algorithm/PyodCBLOF.py b/tods/detection_algorithm/PyodCBLOF.py index 00433f6..4352bd5 100644 --- a/tods/detection_algorithm/PyodCBLOF.py +++ b/tods/detection_algorithm/PyodCBLOF.py @@ -198,12 +198,16 @@ class CBLOFPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hy metadata = metadata_base.PrimitiveMetadata({ "name": "TODS.anomaly_detection_primitives.CBLOFPrimitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_cblof", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.LOCAL_OUTLIER_FACTOR, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATALAB @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", "hyperparams_to_tune": ['contamination'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'CBLOFPrimitive')), }) diff --git a/tods/detection_algorithm/PyodCOF.py b/tods/detection_algorithm/PyodCOF.py index 521502a..3344cd8 100644 --- a/tods/detection_algorithm/PyodCOF.py +++ b/tods/detection_algorithm/PyodCOF.py @@ -35,7 +35,7 @@ from d3m import container, utils as d3m_utils from .UODBasePrimitive import Params_ODBase, Hyperparams_ODBase, UnsupervisedOutlierDetectorBase from pyod.models.cof import COF -# import uuid +import uuid Inputs = d3m_dataframe @@ -105,29 +105,22 @@ class COFPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hype Number of neighbors to use by default for k neighbors queries. """ - __author__ = "Data Lab" - metadata = metadata_base.PrimitiveMetadata( - { + metadata = metadata_base.PrimitiveMetadata({ '__author__' : "DATA Lab at Texas A&M University", 'name': "Connectivity-Based Outlier Factor (COF)", 'python_path': 'd3m.primitives.tods.detection_algorithm.pyod_cof', 'source': { - 'name': 'DATA Lab at Texas A&M University', - 'contact': 'mailto:khlai037@tamu.edu', - 'uris': [ - 'https://gitlab.com/lhenry15/tods.git', - 'https://gitlab.com/lhenry15/tods/-/blob/purav/anomaly-primitives/anomaly_primitives/PyodCOF.py', - ], + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', }, + 'hyperparameters_to_tune':['rank','update','objective','max_iter','learning_rate'], + 'version': '0.0.1', 'algorithm_types': [ - metadata_base.PrimitiveAlgorithmType.PYOD_COF, + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, ], 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - 'id': 'c7259da6-7ce6-42ad-83c6-15238679f5fa', - 'hyperparameters_to_tune':['rank','update','objective','max_iter','learning_rate'], - 'version': '0.0.1', - }, - ) + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'COFPrimitive')), + }) def __init__(self, *, hyperparams: Hyperparams, # diff --git a/tods/detection_algorithm/PyodHBOS.py b/tods/detection_algorithm/PyodHBOS.py index bb8ab4b..c3dc610 100644 --- a/tods/detection_algorithm/PyodHBOS.py +++ b/tods/detection_algorithm/PyodHBOS.py @@ -30,6 +30,7 @@ from d3m.primitive_interfaces.transformer import TransformerPrimitiveBase from d3m.primitive_interfaces.base import ProbabilisticCompositionalityMixin, ContinueFitMixin from d3m import exceptions import pandas +import uuid from d3m import container, utils as d3m_utils @@ -122,17 +123,21 @@ class HBOSPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hyp ``threshold_`` on ``decision_scores_``. """ - __author__: "DATA Lab at Texas A&M University" metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "HBOS", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_hbos", - "source": {'name': 'DATA Lab at Texas A&M University', 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git', 'https://gitlab.com/lhenry15/tods/-/blob/Junjie/anomaly-primitives/anomaly_primitives/PyodHBOS.py']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.HISTOGRAM_BASED_OUTLIER_DETECTION], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "id": "dc722a9f-7621-4900-9b77-7b3b7631ff5e", + "source": { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, "hyperparams_to_tune": ['contamination', 'n_bins', 'alpha', 'tol'], "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'HBOSPrimitive')), }) def __init__(self, *, diff --git a/tods/detection_algorithm/PyodIsolationForest.py b/tods/detection_algorithm/PyodIsolationForest.py index 5c27393..2f4acfe 100644 --- a/tods/detection_algorithm/PyodIsolationForest.py +++ b/tods/detection_algorithm/PyodIsolationForest.py @@ -187,14 +187,19 @@ class IsolationForestPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.IsolationForest", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_iforest", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.ISOLATION_FOREST, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", "hyperparams_to_tune": ['n_estimators', 'contamination'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'IsolationForest')) }) diff --git a/tods/detection_algorithm/PyodKNN.py b/tods/detection_algorithm/PyodKNN.py index cfad177..6df74ea 100644 --- a/tods/detection_algorithm/PyodKNN.py +++ b/tods/detection_algorithm/PyodKNN.py @@ -228,14 +228,19 @@ class KNNPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hype """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.KNNPrimitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_knn", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.K_NEAREST_NEIGHBORS, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATALAB @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", "hyperparams_to_tune": ['n_neighbors', 'method', 'radius', 'algorithm', 'leaf_size', 'p'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'KNNPrimitive')), }) diff --git a/tods/detection_algorithm/PyodLODA.py b/tods/detection_algorithm/PyodLODA.py index cb859a3..230c230 100644 --- a/tods/detection_algorithm/PyodLODA.py +++ b/tods/detection_algorithm/PyodLODA.py @@ -103,15 +103,20 @@ class LODAPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hyp """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.LODAPrimitive", "python_path": "d3m.primitives.anomaly_detection.LODAPrimitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_loda", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.LOCAL_OUTLIER_FACTOR, ], # Wrong - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "version": "0.0.1", + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "hyperparams_to_tune": ['n_bins', 'n_random_cuts', 'contamination'], + "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'LODAPrimitive')), }) diff --git a/tods/detection_algorithm/PyodLOF.py b/tods/detection_algorithm/PyodLOF.py index 96e5147..3a87f6b 100644 --- a/tods/detection_algorithm/PyodLOF.py +++ b/tods/detection_algorithm/PyodLOF.py @@ -207,14 +207,19 @@ class LOFPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hype """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.LOFPrimitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_lof", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.LOCAL_OUTLIER_FACTOR, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "version": "0.0.1", + "source": { + 'name': "DATALAB @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "hyperparams_to_tune": ['n_neighbors', 'algorithm', 'leaf_size', 'p', 'contamination'], + "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'LOFPrimitive')), }) diff --git a/tods/detection_algorithm/PyodOCSVM.py b/tods/detection_algorithm/PyodOCSVM.py index 56212a2..98f6ad3 100644 --- a/tods/detection_algorithm/PyodOCSVM.py +++ b/tods/detection_algorithm/PyodOCSVM.py @@ -198,14 +198,19 @@ class OCSVMPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hy """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.OCSVMPrimitive", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_ocsvm", - "source": {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.MARGIN_CLASSIFIER, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "version": "0.0.1", + "source": { + 'name': "DATA Lab @Taxes A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "hyperparams_to_tune": ['contamination', 'kernel', 'nu', 'gamma', 'degree'], + "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'OCSVMPrimitive')) }) diff --git a/tods/detection_algorithm/PyodSOD.py b/tods/detection_algorithm/PyodSOD.py index b4d69f3..378e22c 100644 --- a/tods/detection_algorithm/PyodSOD.py +++ b/tods/detection_algorithm/PyodSOD.py @@ -114,16 +114,20 @@ class SODPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hype """ metadata = metadata_base.PrimitiveMetadata({ - "__author__": "DATA Lab at Texas A&M University", - "name": "Subspace Outlier Detection Primitive", - "python_path": "d3m.primitives.tods.detection_algorithm.pyod_sod", - "source": {'name': 'DATA Lab at Texas A&M University', 'contact': 'mailto:khlai037@tamu.edu', - 'uris': ['https://gitlab.com/lhenry15/tods.git', 'https://gitlab.com/lhenry15/tods/-/blob/Yile/anomaly-primitives/anomaly_primitives/PyodSOD.py']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.SUBSPACE_OUTLIER_DETECTION], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'SODPrimitive')), - "hyperparams_to_tune": ['contamination', 'n_neighbors', 'ref_set', 'alpha'], - "version": "0.0.1", + "__author__": "DATA Lab at Texas A&M University", + "name": "Subspace Outlier Detection Primitive", + "python_path": "d3m.primitives.tods.detection_algorithm.pyod_sod", + "source": { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, + "hyperparams_to_tune": ['contamination', 'n_neighbors', 'ref_set', 'alpha'], + "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'SODPrimitive')), }) diff --git a/tods/detection_algorithm/PyodSoGaal.py b/tods/detection_algorithm/PyodSoGaal.py index 36392bb..7788c66 100644 --- a/tods/detection_algorithm/PyodSoGaal.py +++ b/tods/detection_algorithm/PyodSoGaal.py @@ -32,6 +32,7 @@ from d3m import exceptions import pandas from d3m import container, utils as d3m_utils +import uuid from .UODBasePrimitive import Params_ODBase, Hyperparams_ODBase, UnsupervisedOutlierDetectorBase from pyod.models.so_gaal import SO_GAAL @@ -158,36 +159,22 @@ class So_GaalPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, """ __author__ = "DATA Lab at Texas A&M University", - metadata = metadata_base.PrimitiveMetadata( - { - 'id': '56e6cfe9-d9e9-495f-83da-cfed6fa27da1', - 'version': '0.1.0', - 'name': 'So_Gaal Anomaly Detection', - 'python_path': 'd3m.primitives.tods.detection_algorithm.pyod_sogaal', - 'keywords': ['Time Series', 'GAN'], - "hyperparams_to_tune": ['stop_epochs','lr_d','lr_g','decay','momentum'], - 'source': { - 'name': 'DATA Lab at Texas A&M University', - 'uris': ['https://gitlab.com/lhenry15/tods.git', - 'https://gitlab.com/lhenry15/tods/-/blob/devesh/tods/detection_algorithm/PyodSoGaal.py'], - 'contact': 'mailto:khlai037@tamu.edu' - - }, - 'installation': [ - {'type': metadata_base.PrimitiveInstallationType.PIP, - 'package_uri': 'git+https://gitlab.com/lhenry15/tods.git@{git_commit}#egg=TODS'.format( - git_commit=d3m_utils.current_git_commit(os.path.dirname(__file__)), - ), - } - - ], - 'algorithm_types': [ - metadata_base.PrimitiveAlgorithmType.DATA_PROFILING, - ], - 'primitive_family': metadata_base.PrimitiveFamily.FEATURE_CONSTRUCTION, - - } - ) + metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", + 'name': 'So_Gaal Anomaly Detection', + 'python_path': 'd3m.primitives.tods.detection_algorithm.pyod_sogaal', + 'source': { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu' + }, + "hyperparams_to_tune": ['contamination', 'stop_epochs','lr_d','lr_g','decay','momentum'], + 'version': '0.1.0', + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + 'primitive_family': metadata_base.PrimitiveFamily.FEATURE_CONSTRUCTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'So_GaalPrimitive')), + }) def __init__(self, *, hyperparams: Hyperparams, # diff --git a/tods/detection_algorithm/PyodVAE.py b/tods/detection_algorithm/PyodVAE.py index 62ae781..0543f78 100644 --- a/tods/detection_algorithm/PyodVAE.py +++ b/tods/detection_algorithm/PyodVAE.py @@ -288,13 +288,19 @@ class VariationalAutoEncoderPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Ou """ metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", "name": "TODS.anomaly_detection_primitives.VariationalAutoEncoder", "python_path": "d3m.primitives.tods.detection_algorithm.pyod_vae", - "source": {'name': "DATA Lab at Texas A&M University", 'contact': 'mailto:khlai037@tamu.edu','uris': ['https://gitlab.com/lhenry15/tods.git']}, - "algorithm_types": [metadata_base.PrimitiveAlgorithmType.VARIATIONAL_AUTO_ENCODER, ], - "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + "source": { + 'name': "DATA Lab at Texas A&M University", + 'contact': 'mailto:khlai037@tamu.edu', + }, "version": "0.0.1", - "hyperparameters_to_tune": [''], + "hyperparameters_to_tune": ['contamination'], + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.ANOMALY_DETECTION, "id": str(uuid.uuid3(uuid.NAMESPACE_DNS, 'AutoEncoderPrimitive')), }) diff --git a/tods/detection_algorithm/SystemWiseDetection.py b/tods/detection_algorithm/SystemWiseDetection.py index d505522..d19b20a 100644 --- a/tods/detection_algorithm/SystemWiseDetection.py +++ b/tods/detection_algorithm/SystemWiseDetection.py @@ -21,6 +21,7 @@ from d3m.container import DataFrame as d3m_dataframe from d3m.metadata import hyperparams, params, base as metadata_base from d3m.base import utils as base_utils +import uuid from d3m.exceptions import PrimitiveNotFittedError __all__ = ('SystemWiseDetectionPrimitive',) @@ -106,36 +107,22 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs, Primitive to find abs_energy of time series """ - __author__ = "DATA Lab at Texas A&M University", - metadata = metadata_base.PrimitiveMetadata( - { - 'id': '3726fa29-28c5-4529-aec5-2f8b4ff2ef9e', - 'version': '0.1.0', - 'name': 'Sytem_Wise_Anomaly_Detection_Primitive', - 'python_path': 'd3m.primitives.tods.detection_algorithm.system_wise_detection', - 'keywords': ['Time Series','Anomalous System '], - "hyperparams_to_tune": ['window_size','method_type','contamination'], - 'source': { - 'name': 'DATA Lab at Texas A&M University', - 'uris': ['https://gitlab.com/lhenry15/tods.git','https://gitlab.com/lhenry15/tods/-/blob/devesh/tods/feature_analysis/StatisticalAbsEnergy.py'], - 'contact': 'mailto:khlai037@tamu.edu' - - }, - 'installation': [ - {'type': metadata_base.PrimitiveInstallationType.PIP, - 'package_uri': 'git+https://gitlab.com/lhenry15/tods.git@{git_commit}#egg=TODS'.format( - git_commit=d3m_utils.current_git_commit(os.path.dirname(__file__)), - ), - } - - ], - 'algorithm_types': [ - metadata_base.PrimitiveAlgorithmType.DATA_PROFILING, - ], - 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - - } - ) + metadata = metadata_base.PrimitiveMetadata({ + "__author__": "DATA Lab at Texas A&M University", + 'name': 'Sytem_Wise_Anomaly_Detection_Primitive', + 'python_path': 'd3m.primitives.tods.detection_algorithm.system_wise_detection', + 'source': { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu' + }, + "hyperparams_to_tune": ['window_size','method_type','contamination'], + 'version': '0.1.0', + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'Sytem_Wise_Anomaly_Detection_Primitive')), + }) def __init__(self, *, hyperparams: Hyperparams) -> None: super().__init__(hyperparams=hyperparams) diff --git a/tods/detection_algorithm/Telemanom.py b/tods/detection_algorithm/Telemanom.py index 12e3aea..06d5160 100644 --- a/tods/detection_algorithm/Telemanom.py +++ b/tods/detection_algorithm/Telemanom.py @@ -35,6 +35,7 @@ from sklearn.utils import check_array # from d3m.primitive_interfaces.base import ProbabilisticCompositionalityMixin, ContinueFitMixin from d3m import exceptions +import uuid # from detection_algorithm.UODBasePrimitive import Params_ODBase, Hyperparams_ODBase, UnsupervisedOutlierDetectorBase @@ -195,29 +196,22 @@ class TelemanomPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params """ - __author__ = "Data Lab" - metadata = metadata_base.PrimitiveMetadata( - { - '__author__' : "DATA Lab at Texas A&M University", - 'name': "Telemanom", - 'python_path': 'd3m.primitives.tods.detection_algorithm.telemanom', - 'source': { - 'name': 'DATA Lab at Texas A&M University', - 'contact': 'mailto:khlai037@tamu.edu', - 'uris': [ - 'https://gitlab.com/lhenry15/tods.git', - 'https://gitlab.com/lhenry15/tods/-/blob/purav/anomaly-primitives/anomaly_primitives/telemanom.py', - ], - }, - 'algorithm_types': [ - metadata_base.PrimitiveAlgorithmType.TELEMANOM, - ], - 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, - 'id': 'c7259da6-7ce6-42ad-83c6-15238679f5fa', - 'hyperparameters_to_tune':['layers','loss_metric','optimizer','epochs','p','l_s','patience','min_delta','dropout','smoothing_perc'], - 'version': '0.0.1', - }, - ) + metadata = metadata_base.PrimitiveMetadata({ + '__author__' : "DATA Lab at Texas A&M University", + 'name': "Telemanom", + 'python_path': 'd3m.primitives.tods.detection_algorithm.telemanom', + 'source': { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, + 'hyperparameters_to_tune':['layers','loss_metric','optimizer','epochs','p','l_s','patience','min_delta','dropout','smoothing_perc'], + 'version': '0.0.1', + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'TelemanomPrimitive')), + }) def __init__(self, *, hyperparams: Hyperparams, # diff --git a/tods/feature_analysis/BKFilter.py b/tods/feature_analysis/BKFilter.py index c252bdc..cb0523b 100644 --- a/tods/feature_analysis/BKFilter.py +++ b/tods/feature_analysis/BKFilter.py @@ -156,20 +156,20 @@ class BKFilterPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hy """ metadata = metadata_base.PrimitiveMetadata({ - "__author__": "DATA Lab at Texas A&M University", - "name": "Baxter-King Filter Primitive", - "python_path": "d3m.primitives.tods.feature_analysis.bk_filter", - "source": { - 'name': 'DATA Lab at Texas A&M University', - 'contact': 'mailto:khlai037@tamu.edu', - }, - "hyperparams_to_tune": ['low', 'high', 'K'], - "version": "0.0.1", - "algorithm_types": [ - metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, - ], - "primitive_family": metadata_base.PrimitiveFamily.FEATURE_CONSTRUCTION, - 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'BKFilterPrimitive')), + "__author__": "DATA Lab at Texas A&M University", + "name": "Baxter-King Filter Primitive", + "python_path": "d3m.primitives.tods.feature_analysis.bk_filter", + "source": { + 'name': 'DATA Lab at Texas A&M University', + 'contact': 'mailto:khlai037@tamu.edu', + }, + "hyperparams_to_tune": ['low', 'high', 'K'], + "version": "0.0.1", + "algorithm_types": [ + metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE, + ], + "primitive_family": metadata_base.PrimitiveFamily.FEATURE_CONSTRUCTION, + 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'BKFilterPrimitive')), })