From cda92ab1c0eb23ab04fa39ec623090bb7128fa94 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Fri, 18 Jun 2021 00:56:12 -0500 Subject: [PATCH 1/5] fix ski unittest --- .../sk_interface/detection_algorithm/test_ski_AutoRegODetector.py | 1 - .../tests/sk_interface/detection_algorithm/test_ski_LSTMODetector.py | 2 +- tods/tests/sk_interface/detection_algorithm/test_ski_PCAODetector.py | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tods/tests/sk_interface/detection_algorithm/test_ski_AutoRegODetector.py b/tods/tests/sk_interface/detection_algorithm/test_ski_AutoRegODetector.py index 90b92c2..78b6bad 100644 --- a/tods/tests/sk_interface/detection_algorithm/test_ski_AutoRegODetector.py +++ b/tods/tests/sk_interface/detection_algorithm/test_ski_AutoRegODetector.py @@ -26,7 +26,6 @@ class AutoRegODetectorSKI_TestCase(unittest.TestCase): self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, contamination=self.contamination, random_state=42) - self.y_test = self.y_test[self.window_size:] self.y_train = self.y_train[self.window_size:] self.transformer = AutoRegODetectorSKI(contamination=self.contamination, window_size=self.window_size) diff --git a/tods/tests/sk_interface/detection_algorithm/test_ski_LSTMODetector.py b/tods/tests/sk_interface/detection_algorithm/test_ski_LSTMODetector.py index bfe11df..4c38131 100644 --- a/tods/tests/sk_interface/detection_algorithm/test_ski_LSTMODetector.py +++ b/tods/tests/sk_interface/detection_algorithm/test_ski_LSTMODetector.py @@ -26,7 +26,7 @@ class LSTMODetectorSKI_TestCase(unittest.TestCase): n_train=self.n_train, n_test=self.n_test, contamination=self.contamination, random_state=42) - self.y_test = self.y_test[1:] + #self.y_test = self.y_test[1:] self.y_train = self.y_train[1:] diff --git a/tods/tests/sk_interface/detection_algorithm/test_ski_PCAODetector.py b/tods/tests/sk_interface/detection_algorithm/test_ski_PCAODetector.py index 6a1127d..723aa80 100644 --- a/tods/tests/sk_interface/detection_algorithm/test_ski_PCAODetector.py +++ b/tods/tests/sk_interface/detection_algorithm/test_ski_PCAODetector.py @@ -22,11 +22,14 @@ class PCAODetectorSKI_TestCase(unittest.TestCase): self.n_test = 100 self.contamination = 0.1 self.roc_floor = 0.0 + self.window_size = 5 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, contamination=self.contamination, random_state=42) - self.transformer = PCAODetectorSKI(contamination=self.contamination) + + self.transformer = PCAODetectorSKI(contamination=self.contamination, window_size=self.window_size) + self.y_test = self.y_test[self.window_size-1:] self.transformer.fit(self.X_train) def test_prediction_labels(self): From 439addcb12a48a16e047a82dbc8a1c13f77a768f Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Fri, 18 Jun 2021 01:18:32 -0500 Subject: [PATCH 2/5] fix ski KDiscord unittest --- .../sk_interface/detection_algorithm/test_ski_KDiscordODetector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tods/tests/sk_interface/detection_algorithm/test_ski_KDiscordODetector.py b/tods/tests/sk_interface/detection_algorithm/test_ski_KDiscordODetector.py index 0c945da..86d3b72 100644 --- a/tods/tests/sk_interface/detection_algorithm/test_ski_KDiscordODetector.py +++ b/tods/tests/sk_interface/detection_algorithm/test_ski_KDiscordODetector.py @@ -22,11 +22,13 @@ class KDiscordODetectorSKI_TestCase(unittest.TestCase): self.n_test = 100 self.contamination = 0.1 self.roc_floor = 0.0 + self.window_size = 5 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, contamination=self.contamination, random_state=42) - self.transformer = KDiscordODetectorSKI(contamination=self.contamination) + self.transformer = KDiscordODetectorSKI(contamination=self.contamination, window_size=self.window_size) + self.y_test = self.y_test[self.window_size-1:] self.transformer.fit(self.X_train) def test_prediction_labels(self): From 87a8aa692f667b85408a366192294a82a77868ca Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Fri, 18 Jun 2021 01:34:19 -0500 Subject: [PATCH 3/5] modify readme --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 142b453..cd92cc6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,9 @@ -# Revisiting Time Series Outlier Detection: Definitions and Benchmarks -## Benchmark Directions -For getting benchmark code, data and result, please follow the instruction below to intall the package and go to the "benchmark/" folder for the details. +# TODS: Automated Time-series Outlier Detection System Logo -[![Build Status](https://travis-ci.org/datamllab/tods.svg?branch=dev)](https://travis-ci.org/datamllab/tods) +[![Build Status](https://travis-ci.org/datamllab/tods.svg?branch=master)](https://travis-ci.org/datamllab/tods) TODS is a full-stack automated machine learning system for outlier detection on multivariate time-series data. TODS provides exhaustive modules for building machine learning-based outlier detection systems, including: data processing, time series processing, feature analysis (extraction), detection algorithms, and reinforcement module. The functionalities provided via these modules include data preprocessing for general purposes, time series data smoothing/transformation, extracting features from time/frequency domains, various detection algorithms, and involving human expertise to calibrate the system. Three common outlier detection scenarios on time-series data can be performed: point-wise detection (time points as outliers), pattern-wise detection (subsequences as outliers), and system-wise detection (sets of time series as outliers), and a wide-range of corresponding algorithms are provided in TODS. This package is developed by [DATA Lab @ Texas A&M University](https://people.engr.tamu.edu/xiahu/index.html). From 9d590eddb9cc217f880fdbade88ed85ad7625682 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Sat, 19 Jun 2021 22:56:58 -0500 Subject: [PATCH 4/5] modify readme --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cd92cc6..93b153f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -# TODS: Automated Time-series Outlier Detection System +# Revisiting Time Series Outlier Detection: Definition and Benchmark +* Install the TODS package follow the instruction below. +* Go to [benchmark](https://github.com/datamllab/tods/tree/benchmark/benchmark) folder and follow the instructions to run the experiments. Logo + +# TODS: Automated Time-series Outlier Detection System [![Build Status](https://travis-ci.org/datamllab/tods.svg?branch=master)](https://travis-ci.org/datamllab/tods) TODS is a full-stack automated machine learning system for outlier detection on multivariate time-series data. TODS provides exhaustive modules for building machine learning-based outlier detection systems, including: data processing, time series processing, feature analysis (extraction), detection algorithms, and reinforcement module. The functionalities provided via these modules include data preprocessing for general purposes, time series data smoothing/transformation, extracting features from time/frequency domains, various detection algorithms, and involving human expertise to calibrate the system. Three common outlier detection scenarios on time-series data can be performed: point-wise detection (time points as outliers), pattern-wise detection (subsequences as outliers), and system-wise detection (sets of time series as outliers), and a wide-range of corresponding algorithms are provided in TODS. This package is developed by [DATA Lab @ Texas A&M University](https://people.engr.tamu.edu/xiahu/index.html). From 3d7971fa8b4190d56e5854535e15f672f781015a Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Tue, 6 Jul 2021 19:54:41 -0700 Subject: [PATCH 5/5] fix AutoReg --- tods/detection_algorithm/core/MultiAutoRegOD.py | 6 ++++++ tods/tests/detection_algorithm/test_AutoRegODetect.py | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tods/detection_algorithm/core/MultiAutoRegOD.py b/tods/detection_algorithm/core/MultiAutoRegOD.py index 9ef92b3..01834bc 100644 --- a/tods/detection_algorithm/core/MultiAutoRegOD.py +++ b/tods/detection_algorithm/core/MultiAutoRegOD.py @@ -154,6 +154,12 @@ class MultiAutoRegOD(CollectiveBaseDetector): self.decision_scores_ = self._score_combination( self._decison_mat_scalaled) + # print(self.decision_scores_.shape, self.left_inds_.shape, self.right_inds_.shape) + self.decision_scores_ = np.concatenate((np.zeros((self.window_size,)), self.decision_scores_)) + self.left_inds_ = np.concatenate(((-self.window_size) * np.ones((self.window_size,)).astype(np.int), self.left_inds_)) + self.right_inds_ = np.concatenate((np.zeros((self.window_size,)).astype(np.int), self.right_inds_)) + # print(self.decision_scores_.shape, self.left_inds_.shape, self.right_inds_.shape) + self._process_decision_scores() return self diff --git a/tods/tests/detection_algorithm/test_AutoRegODetect.py b/tods/tests/detection_algorithm/test_AutoRegODetect.py index 759aabc..24cadf7 100644 --- a/tods/tests/detection_algorithm/test_AutoRegODetect.py +++ b/tods/tests/detection_algorithm/test_AutoRegODetect.py @@ -28,10 +28,10 @@ class AutoRegODetectTestCase(unittest.TestCase): self.X_train = d3m_dataframe({'data': [3., 4., 8., 16, 18, 13., 22., 36., 59., 128, 62, 67, 78, 100]}, columns=['data'], generate_metadata=True) - self.y_train = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + self.y_train = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) self.X_test = d3m_dataframe({'data': [3., 4., 8.6, 13.4, 22.5, 17, 19.2, 36.1, 127, -23, 59.2]}, columns=['data'], generate_metadata=True) - self.y_test = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0]) + self.y_test = np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0]) @@ -46,6 +46,8 @@ class AutoRegODetectTestCase(unittest.TestCase): self.prediction_labels = self.primitive.produce(inputs=self.X_test).value self.prediction_score = self.primitive.produce_score(inputs=self.X_test).value + # print(self.prediction_labels) + self.collective_common_test = CollectiveCommonTest(model=self.primitive._clf, X_train=self.X_train, y_train=self.y_train,