System Wise detection added

Former-commit-id: e4302b5301 [formerly 4aaa7e334e] [formerly 6f539539fa [formerly f88384aa0a]] [formerly 9d6ef41d61 [formerly af05d0388a] [formerly 98a735f030 [formerly 4d7c25c428]]] [formerly 83fb6a49f3 [formerly a8357e514e] [formerly 8d2001a47a [formerly 4da78f5507]] [formerly 2e5ca864e3 [formerly 15d36e9ea0] [formerly 9f3c47c7b1 [formerly cbfccef799]]]] [formerly 6eaafdda6d [formerly 5943f5a18d] [formerly 4a72e151d4 [formerly dea240cb7c]] [formerly 832b5a2819 [formerly b59d6be50a] [formerly b6e28bbdf3 [formerly 4d2c50b212]]] [formerly e4b2e4ad8f [formerly dcb324a202] [formerly 0d3d65d68b [formerly 7cd9cec3f0]] [formerly 5ea92c8534 [formerly 77f44ab716] [formerly 4b866c4ce5 [formerly 53a6a05c6f]]]]] [formerly e309d0478e [formerly 5e7023951a] [formerly 5211ed45f0 [formerly f95a829607]] [formerly fcfeed2bad [formerly 86004bdf7f] [formerly ec61732826 [formerly a1d353f7fc]]] [formerly 2c7e553943 [formerly 4d69729004] [formerly ccb35644eb [formerly 5f7775ed42]] [formerly 5d250477e3 [formerly d169bd2ad4] [formerly 0d0c9f6b03 [formerly 5487a767bf]]]] [formerly 79af600ff9 [formerly f14c510112] [formerly 258d1abf33 [formerly 5928649d91]] [formerly 59c329f4b0 [formerly 77391acc47] [formerly de7b6992b7 [formerly 77d5517d42]]] [formerly 5cdaaecda4 [formerly 077c1200fa] [formerly 6b3e082e3f [formerly 6d547b0ea9]] [formerly 7d4e1f6bd5 [formerly 3f9f8bf1e9] [formerly 3ab33e4e0a [formerly 15c823899c]]]]]]
Former-commit-id: 7d581a11ae [formerly 8e7111350d] [formerly e470c42ba3 [formerly 04e4737911]] [formerly 9492392d98 [formerly f7baa50dac] [formerly ea939f8031 [formerly 499e514508]]] [formerly 9e31983ac7 [formerly 50823179fc] [formerly 8e53f8ffb3 [formerly 1b79318b09]] [formerly 5057c32d67 [formerly 8c89e00fc7] [formerly f7ac6d7a45 [formerly 44e943805d]]]] [formerly e63ff23c4b [formerly d769223bc3] [formerly e560da8156 [formerly c493de91c1]] [formerly a91671bb44 [formerly a19cc66d99] [formerly a61ccd6995 [formerly f738fbeca6]]] [formerly 163b1d74e1 [formerly ebbf736e53] [formerly 4f973dd70a [formerly 7ecc4fb1d5]] [formerly 5f369aa799 [formerly 7ba6e4367d] [formerly 3ab33e4e0a]]]]
Former-commit-id: 4bf5312685 [formerly ed91c6d8a1] [formerly 3ea0046108 [formerly 84bfff7021]] [formerly 520135ee97 [formerly 186c07386d] [formerly 20a9473215 [formerly de476ee28c]]] [formerly 4130e0a401 [formerly 1c82429fa5] [formerly 2a8a9e3be8 [formerly 0120401179]] [formerly 10d866fe71 [formerly 0a7d02b841] [formerly fc8c90a937 [formerly 29c5b1f03d]]]]
Former-commit-id: 684b9d1af0 [formerly 99ed1e909e] [formerly e7b2f2759c [formerly 0926101681]] [formerly 9c854f8bf8 [formerly a90c5f05a1] [formerly 77b126bf28 [formerly 6a7e797857]]]
Former-commit-id: 48133573d7 [formerly 50a7456017] [formerly b28a535153 [formerly e0d06b8061]]
Former-commit-id: 679a59d059 [formerly e4bb574907]
Former-commit-id: 3022920ffe

tods/detection_algorithm/SystemWiseDetection.py

@@ -38,13 +38,13 @@ class Hyperparams(hyperparams.Hyperparams):
 
        #Tuning Parameter
        #default -1 considers entire time series is considered
-       window_size = hyperparams.Hyperparameter(default=-1, semantic_types=[
+       window_size = hyperparams.Hyperparameter(default=10, semantic_types=[
            'https://metadata.datadrivendiscovery.org/types/TuningParameter',
        ], description="Window Size for decomposition")
 
        method_type = hyperparams.Enumeration(
-           values=['max', 'avg', 'majority_voting_sum'],
-           default='avg',
+           values=['max', 'avg', 'sliding_window_sum','majority_voting_sliding_window_sum','majority_voting_sliding_window_max'],
+           default='majority_voting_sliding_window_max',
            semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'],
            description="The type of method used to find anomalous system",
        )
@@ -338,10 +338,13 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs,
 
         transformed_X = []
         if(method_type=="max"):
+            """
+            Sytems are sorted based on maximum of reconstruction errors"
+            """
             maxOutlierScorePerSystemList = []
             for systemId in systemIds:
                 systemDf = groupedX.get_group(systemId)
-                maxOutlierScorePerSystemList.append(np.max(systemDf["value_0"].values))
+                maxOutlierScorePerSystemList.append(np.max(np.abs(systemDf["value_0"].values)))
 
             ranking = np.sort(maxOutlierScorePerSystemList)
             threshold = ranking[int((1 - contamination) * len(ranking))]
@@ -353,10 +356,40 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs,
                 transformed_X.append([systemIds[iter],ranking[iter]])
 
         if (method_type == "avg"):
-            maxOutlierScorePerSystemList = []
+            """
+            Sytems are sorted based on average of reconstruction errors"
+            """
+            avgOutlierScorePerSystemList = []
+            for systemId in systemIds:
+                systemDf = groupedX.get_group(systemId)
+                avgOutlierScorePerSystemList.append(np.mean(np.abs(systemDf["value_0"].values)))
+
+            ranking = np.sort(avgOutlierScorePerSystemList)
+            threshold = ranking[int((1 - contamination) * len(ranking))]
+            self.threshold = threshold
+            mask = (avgOutlierScorePerSystemList >= threshold)
+            ranking[mask] = 1
+            ranking[np.logical_not(mask)] = 0
+            for iter in range(len(systemIds)):
+                transformed_X.append([systemIds[iter], ranking[iter]])
+
+        if (method_type == "sliding_window_sum"):
+            """
+            Sytems are sorted based on max of max of reconstruction errors in each window"
+            """
+            OutlierScorePerSystemList = []
             for systemId in systemIds:
                 systemDf = groupedX.get_group(systemId)
-                maxOutlierScorePerSystemList.append(np.mean(np.abs(systemDf["value_0"].values)))
+                column_value = systemDf["value_0"].values
+                column_score = np.zeros(len(column_value))
+                for iter in range(window_size - 1, len(column_value)):
+                    sequence = column_value[iter - window_size + 1:iter + 1]
+                    column_score[iter] = np.sum(np.abs(sequence))
+                column_score[:window_size - 1] = column_score[window_size - 1]
+                OutlierScorePerSystemList.append(column_score.tolist())
+            OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList)
+
+            maxOutlierScorePerSystemList = OutlierScorePerSystemList.max(axis=1).tolist()
 
             ranking = np.sort(maxOutlierScorePerSystemList)
             threshold = ranking[int((1 - contamination) * len(ranking))]
@@ -367,6 +400,63 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs,
             for iter in range(len(systemIds)):
                 transformed_X.append([systemIds[iter], ranking[iter]])
 
+        if (method_type == "majority_voting_sliding_window_sum"):
+            """
+            Sytem with most vote based on max of sum of reconstruction errors in each window
+            """
+            OutlierScorePerSystemList = []
+            for systemId in systemIds:
+                systemDf = groupedX.get_group(systemId)
+                column_value = systemDf["value_0"].values
+                column_score = np.zeros(len(column_value))
+                for iter in range(window_size - 1, len(column_value)):
+                    sequence = column_value[iter - window_size + 1:iter + 1]
+                    column_score[iter] = np.sum(np.abs(sequence))
+                column_score[:window_size - 1] = column_score[window_size - 1]
+                OutlierScorePerSystemList.append(column_score.tolist())
+            OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList)
+            OutlierScorePerSystemList = (
+                    OutlierScorePerSystemList == OutlierScorePerSystemList.max(axis=0)[None, :]).astype(int)
+
+            maxOutlierScorePerSystemList = OutlierScorePerSystemList.sum(axis=1).tolist()
+
+            ranking = np.sort(maxOutlierScorePerSystemList)
+            threshold = ranking[int((1 - contamination) * len(ranking))]
+            self.threshold = threshold
+            mask = (maxOutlierScorePerSystemList >= threshold)
+            ranking[mask] = 1
+            ranking[np.logical_not(mask)] = 0
+            for iter in range(len(systemIds)):
+                transformed_X.append([systemIds[iter], ranking[iter]])
+
+        if (method_type == "majority_voting_sliding_window_max"):
+            """
+            Sytem with most vote based on max of max of reconstruction errors in each window
+            """
+            OutlierScorePerSystemList = []
+            for systemId in systemIds:
+                systemDf = groupedX.get_group(systemId)
+                column_value = systemDf["value_0"].values
+                column_score = np.zeros(len(column_value))
+                for iter in range(window_size - 1, len(column_value)):
+                    sequence = column_value[iter - window_size + 1:iter + 1]
+                    column_score[iter] = np.max(np.abs(sequence))
+                column_score[:window_size - 1] = column_score[window_size - 1]
+                OutlierScorePerSystemList.append(column_score.tolist())
+            OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList)
+            OutlierScorePerSystemList = (
+                    OutlierScorePerSystemList == OutlierScorePerSystemList.max(axis=0)[None, :]).astype(int)
+
+            maxOutlierScorePerSystemList = OutlierScorePerSystemList.sum(axis=1).tolist()
+
+            ranking = np.sort(maxOutlierScorePerSystemList)
+            threshold = ranking[int((1 - contamination) * len(ranking))]
+            self.threshold = threshold
+            mask = (maxOutlierScorePerSystemList >= threshold)
+            ranking[mask] = 1
+            ranking[np.logical_not(mask)] = 0
+            for iter in range(len(systemIds)):
+                transformed_X.append([systemIds[iter], ranking[iter]])
 
         return transformed_X