From c36004451805cf65e7aa2d2c0d828df84a0e1808 Mon Sep 17 00:00:00 2001 From: Devesh Kumar Date: Tue, 29 Sep 2020 08:12:39 -0500 Subject: [PATCH] System Wise detection added Former-commit-id: e4302b53012091459b225210d586ba093e837480 [formerly 4aaa7e334e3670203277fde693162e92a4cf9d35] [formerly 6f539539faa232b1f6c1a539c2fc0636ad31058d [formerly f88384aa0a8126b80882c17a24ae14d61025c876]] [formerly 9d6ef41d61d601e97f10c4e76d29120cecf65efd [formerly af05d0388a4383f969a72fdd1d97733e2a8eff77] [formerly 98a735f03051db87738b6182e40ba1250fce07fe [formerly 4d7c25c4286af60d41c3ef2d973894c57019e521]]] [formerly 83fb6a49f3c2b8c242b764fbb3951ac7ba148926 [formerly a8357e514ed743af0fd3a179b7fcdb65192bf4a9] [formerly 8d2001a47aeb430f1d1d4893e8202da809bd32cd [formerly 4da78f5507915fb21d44d060b411b53f4496b54e]] [formerly 2e5ca864e36caa4bf63d71272cb6a104fcf96bde [formerly 15d36e9ea0f6fcb203f8b5a8ebf50aabf5d8f33b] [formerly 9f3c47c7b1489ac97ef329b18f3d1142107417bc [formerly cbfccef799011cecbb1593a7f10c8425254366ed]]]] [formerly 6eaafdda6d0f87ec3f9c6bdd400d431212f81caf [formerly 5943f5a18d08b2ef584f95ca077a5f40ed74fd2c] [formerly 4a72e151d4751d18dd414c735d2b93adf82cc8b0 [formerly dea240cb7cf33f99d5316f7ab6c7faec9667b0eb]] [formerly 832b5a2819689cafdf14c3009b8be5b5fc6fed70 [formerly b59d6be50aa90336a420323b092596d625f525ec] [formerly b6e28bbdf39ce981c92dc3118cb476226bb1623f [formerly 4d2c50b21206b52aaa993e10ca7277c0990532f5]]] [formerly e4b2e4ad8f7ac6c516dc3a300661411659bb2694 [formerly dcb324a2027077657b6fa653ac559a8ffcd504b0] [formerly 0d3d65d68b064f119f59620c7b15c64b2046a44a [formerly 7cd9cec3f0d1d3f8283386e9b7a97ee2f86dcfb3]] [formerly 5ea92c8534a1cc4f546d7f61acf3b0fd45dc3c73 [formerly 77f44ab71678f1fcec159e641f861e48906bd23a] [formerly 4b866c4ce539335b39a4b7c55589be1fa8f1737f [formerly 53a6a05c6fa8af9375374da99d4dd870e2d03edd]]]]] [formerly e309d0478ef1df407339c1e95df86e8cf0e30d63 [formerly 5e7023951a6bae3900bc52e078d7e2f7b68de4db] [formerly 5211ed45f021801b4eccb943dd7c275b9ae7be0a [formerly f95a8296075f1c1af00a821094e64b3460499190]] [formerly fcfeed2bad85c1af622dc838dd23f194a98a2873 [formerly 86004bdf7f6a0de1171dd3206a129827fd86184e] [formerly ec61732826c6334943b19a4cba3065569cbebeb5 [formerly a1d353f7fcc0ffb9ec7b4b34ff1737ec54b6925b]]] [formerly 2c7e5539438d15a3d86416bab4dd41cec502bc64 [formerly 4d69729004cf788bcf43c884cc93042125f687b8] [formerly ccb35644eb05984c55c4f421de8490e2627984da [formerly 5f7775ed4285ffe3e66a155e727bc76db6e03c0d]] [formerly 5d250477e36092648e51df1b606f4c808dc2a77b [formerly d169bd2ad41b8c8f7f4e819ac5a840cb614e1a73] [formerly 0d0c9f6b03057eb52accb2db6e9e3823839f021d [formerly 5487a767bf398bb866590d7c53feb2ce256ce27f]]]] [formerly 79af600ff94dbc2cb149aea473ea09882439db00 [formerly f14c51011228ed0e4ef21e89900b1ca8390c3d8d] [formerly 258d1abf33cb88fe1b4fce32bbab9ac19fbcc03a [formerly 5928649d9155ef83a217c063de46dc079aa17c79]] [formerly 59c329f4b01ab8384185fd694dbaa2cda616eded [formerly 77391acc479ca7346b17bb31ef9c7957da4eb72c] [formerly de7b6992b77cdbe74882588043eff754f0d19102 [formerly 77d5517d4215c006abe2d1370463cfa983478ff9]]] [formerly 5cdaaecda44c680e2a8a5444191a27f66e47f7da [formerly 077c1200fa0d943c144d0e330efe9ae69b63be45] [formerly 6b3e082e3fc0d85d6786b25408e068b8389fe3d7 [formerly 6d547b0ea9620bc3b483c7c677a3e365d40cb8e3]] [formerly 7d4e1f6bd549ba644fa0881a015f29ba20317b23 [formerly 3f9f8bf1e98d096814fe40207b07a16d8bc050a4] [formerly 3ab33e4e0a225de440dfb5acb2a8a1e1746e7345 [formerly 15c823899c80e0f56762f36037c061649cb95d18]]]]]] Former-commit-id: 7d581a11ae023631ac1cf46034f25c1dab12a958 [formerly 8e7111350dd185b524cc3054a3019ee107fea0e6] [formerly e470c42ba3e5387bc62170d506d4d2c1140604af [formerly 04e47379117cca2eac6f0dda8e12ec8133d461e0]] [formerly 9492392d985595878306401aff9084158daa941f [formerly f7baa50dacafe68372e093799dbe4be1da7638e5] [formerly ea939f8031b667fa7a498c740ea85cf9506c9fd7 [formerly 499e5145084a9359c29cd29f16e230a89bc779b7]]] [formerly 9e31983ac792b01db00e28f52d25795be6a6675b [formerly 50823179fca9f453893364acb3f32891c0f82acd] [formerly 8e53f8ffb30f851954cc2d77c27fb05eb2a6e828 [formerly 1b79318b09503dc74c51f34a0a097f3b5f24eeff]] [formerly 5057c32d67ede6c06bf1b27105da611d21b9fe1c [formerly 8c89e00fc70a46ceb758984904e5b5cf994e9257] [formerly f7ac6d7a450e873b6816411a307f80517f19b43c [formerly 44e943805dc5b3ae29b75c31753f9075d455cf60]]]] [formerly e63ff23c4b761af8884cf0a39b3b2dd121baf947 [formerly d769223bc370b065a8fd694e2d8c727ca08ce142] [formerly e560da815613e0ad438c334ea9cf5fb6f2fcd515 [formerly c493de91c119d4ddc5d911d43acf2deaf8178f94]] [formerly a91671bb4490b9db3bc8beb1156e0db62654bb88 [formerly a19cc66d9978428972e648655be6bc6db9752daa] [formerly a61ccd69958257e351d10665b5d087476291e0d8 [formerly f738fbeca6592684e22b21c22614e3f3c82f1e7a]]] [formerly 163b1d74e1265eea978861789d95184194b84db0 [formerly ebbf736e533fcaf98ef2184797cbd566a891fdad] [formerly 4f973dd70a339853fbbb9c3de946917b5f7cce84 [formerly 7ecc4fb1d5073c63b8ab04a57d7a039ffcd6d903]] [formerly 5f369aa799b1bc0252ea17361e7664b3f136fa26 [formerly 7ba6e4367d74f807a75ae61221895502e8800618] [formerly 3ab33e4e0a225de440dfb5acb2a8a1e1746e7345]]]] Former-commit-id: 4bf5312685ac1e81a43d78699ea906e9e9e6af71 [formerly ed91c6d8a1a0c6eb5d5c0f0b1207f2a83bc7c4ff] [formerly 3ea0046108dc66ef66a66540d38b586d719e129a [formerly 84bfff70217c36dc6bf65981bc2e77681588bc69]] [formerly 520135ee9765a414470af3fade617f92ef16d0a7 [formerly 186c07386d43be2b52d2ed66459b1ea22f8f5a76] [formerly 20a947321569ab2f8d0c1e97f2d23131f8d1d77c [formerly de476ee28c8908cd872766fce04fc8824358c9c3]]] [formerly 4130e0a401904d12774831a1b25cdad09e29a322 [formerly 1c82429fa596438094969643105328d88fc06e76] [formerly 2a8a9e3be83297f4eb9cab1c63a40e152ee75b93 [formerly 0120401179eb347d2ae1d397d69e87cb943a7e08]] [formerly 10d866fe7177973257b90753d2350b6ccb7c34a3 [formerly 0a7d02b841e47dbe455b5a61f6edfd337ebcd57b] [formerly fc8c90a9376a73e18cd104c42d22261908663063 [formerly 29c5b1f03d7eb42dc0f3bc17f78951e1cdad818a]]]] Former-commit-id: 684b9d1af033eec77c562fb9cadf72efd2ffb366 [formerly 99ed1e909ede98259f00b68ad6da9c5abf1aab64] [formerly e7b2f2759cfb65bc64b912141acbf7375e4a3251 [formerly 09261016819cf039b6cff3185d6fc547eba7aa46]] [formerly 9c854f8bf887f1b87babf71ab550184a00a74ee5 [formerly a90c5f05a1edce6527e9dd1103b36549893f4dc4] [formerly 77b126bf28439c1aca1444d7c19136bffcb2e2b5 [formerly 6a7e79785726dd1ce2fd733b15c732cd74fea4b0]]] Former-commit-id: 48133573d756ad5ef6d81316f7c79db9e884cfcf [formerly 50a7456017ecd50481ea6945211b00180150039c] [formerly b28a5351535dfaae12a3439b9e8ddc3344b2bec5 [formerly e0d06b8061e7de8b0c4e79a79c201cc873695534]] Former-commit-id: 679a59d059f47623a7c649332569af47b285e6e5 [formerly e4bb5749072c6b19592c0eca750d46c41fb37126] Former-commit-id: 3022920ffe453a5433d74c13cc039de9c095d919 --- tods/detection_algorithm/SystemWiseDetection.py | 102 ++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 6 deletions(-) diff --git a/tods/detection_algorithm/SystemWiseDetection.py b/tods/detection_algorithm/SystemWiseDetection.py index 3cb122d..d505522 100644 --- a/tods/detection_algorithm/SystemWiseDetection.py +++ b/tods/detection_algorithm/SystemWiseDetection.py @@ -38,13 +38,13 @@ class Hyperparams(hyperparams.Hyperparams): #Tuning Parameter #default -1 considers entire time series is considered - window_size = hyperparams.Hyperparameter(default=-1, semantic_types=[ + window_size = hyperparams.Hyperparameter(default=10, semantic_types=[ 'https://metadata.datadrivendiscovery.org/types/TuningParameter', ], description="Window Size for decomposition") method_type = hyperparams.Enumeration( - values=['max', 'avg', 'majority_voting_sum'], - default='avg', + values=['max', 'avg', 'sliding_window_sum','majority_voting_sliding_window_sum','majority_voting_sliding_window_max'], + default='majority_voting_sliding_window_max', semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], description="The type of method used to find anomalous system", ) @@ -338,10 +338,13 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs, transformed_X = [] if(method_type=="max"): + """ + Sytems are sorted based on maximum of reconstruction errors" + """ maxOutlierScorePerSystemList = [] for systemId in systemIds: systemDf = groupedX.get_group(systemId) - maxOutlierScorePerSystemList.append(np.max(systemDf["value_0"].values)) + maxOutlierScorePerSystemList.append(np.max(np.abs(systemDf["value_0"].values))) ranking = np.sort(maxOutlierScorePerSystemList) threshold = ranking[int((1 - contamination) * len(ranking))] @@ -353,10 +356,40 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs, transformed_X.append([systemIds[iter],ranking[iter]]) if (method_type == "avg"): - maxOutlierScorePerSystemList = [] + """ + Sytems are sorted based on average of reconstruction errors" + """ + avgOutlierScorePerSystemList = [] + for systemId in systemIds: + systemDf = groupedX.get_group(systemId) + avgOutlierScorePerSystemList.append(np.mean(np.abs(systemDf["value_0"].values))) + + ranking = np.sort(avgOutlierScorePerSystemList) + threshold = ranking[int((1 - contamination) * len(ranking))] + self.threshold = threshold + mask = (avgOutlierScorePerSystemList >= threshold) + ranking[mask] = 1 + ranking[np.logical_not(mask)] = 0 + for iter in range(len(systemIds)): + transformed_X.append([systemIds[iter], ranking[iter]]) + + if (method_type == "sliding_window_sum"): + """ + Sytems are sorted based on max of max of reconstruction errors in each window" + """ + OutlierScorePerSystemList = [] for systemId in systemIds: systemDf = groupedX.get_group(systemId) - maxOutlierScorePerSystemList.append(np.mean(np.abs(systemDf["value_0"].values))) + column_value = systemDf["value_0"].values + column_score = np.zeros(len(column_value)) + for iter in range(window_size - 1, len(column_value)): + sequence = column_value[iter - window_size + 1:iter + 1] + column_score[iter] = np.sum(np.abs(sequence)) + column_score[:window_size - 1] = column_score[window_size - 1] + OutlierScorePerSystemList.append(column_score.tolist()) + OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList) + + maxOutlierScorePerSystemList = OutlierScorePerSystemList.max(axis=1).tolist() ranking = np.sort(maxOutlierScorePerSystemList) threshold = ranking[int((1 - contamination) * len(ranking))] @@ -367,6 +400,63 @@ class SystemWiseDetectionPrimitive(transformer.TransformerPrimitiveBase[Inputs, for iter in range(len(systemIds)): transformed_X.append([systemIds[iter], ranking[iter]]) + if (method_type == "majority_voting_sliding_window_sum"): + """ + Sytem with most vote based on max of sum of reconstruction errors in each window + """ + OutlierScorePerSystemList = [] + for systemId in systemIds: + systemDf = groupedX.get_group(systemId) + column_value = systemDf["value_0"].values + column_score = np.zeros(len(column_value)) + for iter in range(window_size - 1, len(column_value)): + sequence = column_value[iter - window_size + 1:iter + 1] + column_score[iter] = np.sum(np.abs(sequence)) + column_score[:window_size - 1] = column_score[window_size - 1] + OutlierScorePerSystemList.append(column_score.tolist()) + OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList) + OutlierScorePerSystemList = ( + OutlierScorePerSystemList == OutlierScorePerSystemList.max(axis=0)[None, :]).astype(int) + + maxOutlierScorePerSystemList = OutlierScorePerSystemList.sum(axis=1).tolist() + + ranking = np.sort(maxOutlierScorePerSystemList) + threshold = ranking[int((1 - contamination) * len(ranking))] + self.threshold = threshold + mask = (maxOutlierScorePerSystemList >= threshold) + ranking[mask] = 1 + ranking[np.logical_not(mask)] = 0 + for iter in range(len(systemIds)): + transformed_X.append([systemIds[iter], ranking[iter]]) + + if (method_type == "majority_voting_sliding_window_max"): + """ + Sytem with most vote based on max of max of reconstruction errors in each window + """ + OutlierScorePerSystemList = [] + for systemId in systemIds: + systemDf = groupedX.get_group(systemId) + column_value = systemDf["value_0"].values + column_score = np.zeros(len(column_value)) + for iter in range(window_size - 1, len(column_value)): + sequence = column_value[iter - window_size + 1:iter + 1] + column_score[iter] = np.max(np.abs(sequence)) + column_score[:window_size - 1] = column_score[window_size - 1] + OutlierScorePerSystemList.append(column_score.tolist()) + OutlierScorePerSystemList = np.asarray(OutlierScorePerSystemList) + OutlierScorePerSystemList = ( + OutlierScorePerSystemList == OutlierScorePerSystemList.max(axis=0)[None, :]).astype(int) + + maxOutlierScorePerSystemList = OutlierScorePerSystemList.sum(axis=1).tolist() + + ranking = np.sort(maxOutlierScorePerSystemList) + threshold = ranking[int((1 - contamination) * len(ranking))] + self.threshold = threshold + mask = (maxOutlierScorePerSystemList >= threshold) + ranking[mask] = 1 + ranking[np.logical_not(mask)] = 0 + for iter in range(len(systemIds)): + transformed_X.append([systemIds[iter], ranking[iter]]) return transformed_X