From 24e5e677b7c7c0e39c85cdd3d071f0583ae0d9eb Mon Sep 17 00:00:00 2001 From: JUNJIE XU Date: Tue, 13 Oct 2020 21:50:46 +0800 Subject: [PATCH] coverall Former-commit-id: 846b08ee7c7231b6d7c14a2f438e87dfcb1031e5 [formerly 898f9492d1265a61378f0024bd85e7390c7827a8] [formerly 3ba67f75a03fbdd98ee1d739401c7e70be501e0e [formerly 58b375e8e7d4402192456eddf61b31290ad4c843]] [formerly ed311ab694c8a7976dfd2d3ce19d37633c76bb7f [formerly 4f094bb67dccbb4d2bdcc1a52e27fc905322ac05] [formerly f26d423c1657dd86a80d62db0910a5a28026fd7d [formerly f794ff34ef5a653073cafa8fc32a86d3ba349d6f]]] [formerly f05635c1f4778354851f77dd2b865e5868aa802c [formerly 79b26421d1ba405b9a5e52820d1c7c82141c441f] [formerly 58d01d07652b5244d2ab9c4f443ba7e476fa7dd5 [formerly ac8fad3853e11e7e3aeae5bc3d7bac438ec9f4e3]] [formerly b0b655300fcb4a66dba33048fe107e0f04dc80b2 [formerly 9726f96eb44e42b4fe69b4128fdf28886f4d2408] [formerly c28b652837994041a874dddb07a3505eb92bd336 [formerly b66948d1d013c1b6c49f42651608c60f06fa55a1]]]] [formerly 464b9b56a8c11530dde979fe4bdf243cd845870b [formerly 6d27652e8405885fda067d889bdd5699d46dfda4] [formerly f8cdb514b123e8834c9bda912d8b36fab00baab9 [formerly 36da7b0133735fb5fb8146517e6cfccaa7995fae]] [formerly 58f0a2ad3bd296c787a3442de7f6c666ed9343b3 [formerly ae260a8a2c566ec9681892e384aa65ff277dc1b4] [formerly 0b4306ce91fb3a292944a4c6ddb073f0cf7f50e4 [formerly 4736c33b8206edb730cd6c76ea496a2f29af9522]]] [formerly 15024cfb98cef75c0cce1c2e3cf57795fb3bff4e [formerly affb9d5be6bd27b328369b29dc3d1ed4b19a46cc] [formerly 49b5cbb379b2d102c4d05fd6ac4e5513d588105d [formerly d386e84f3ff5464ddb0ed1cb3c5fc720504d3bcc]] [formerly 0df3fd4bc89ac170c90aff597baeb26537e4f2fe [formerly 1452be20a9e0fa96f7e2f29ad76c0c34c6f40aa4] [formerly 6cea47368ccb564b59b7ff6515a3059c057a4a4c [formerly 4beb72e53ded32aa7f32c22442a4cddfc76dcee6]]]]] [formerly 2cb122a0e2b77df9b997422a338cff5a58675795 [formerly 369a07af54ba4095585522aa5302cc9166946505] [formerly 117719f457b878b4dc14bb91c3b1e1d6fbfda83e [formerly f418d8f99e673dd001bc90862281651ade3070bf]] [formerly a2f8ef45a1b7b2a0b6e922c01759b29bda953fb8 [formerly 0f9187e9eb2d8e5aecf79dc5be27fa12fa6a32b9] [formerly 1e667b994f6fbbf93a51a956ab65d14b1cfc52ab [formerly 684958c8284454114253daa7df6b4c29df391112]]] [formerly 353202c937a50f79a92be110ef4d2b86c1427e0c [formerly 2e4d2604324f75f2e44fbdce9b487a682915d460] [formerly 0f060e89f2d19a6c4fa6159c85ffc79875f47393 [formerly 12d6c71af714be6a4bc025d0a67fc889bf4f4558]] [formerly f90287aa507629bd4ef5b35d4378f330a00e9b62 [formerly 5f14e91aaabb3b567f10767d5ada5beeea152725] [formerly 15e8680f49bcce301a81f7ca91c20918c09bdee9 [formerly 97f0a20a9f6a1cd010a85ddbdfa35d38fc4ae660]]]] [formerly 4188a6742e47717addc1f4739c20d87f056ca5f2 [formerly 4f52141057a049e420321e3394dc24e8faf3594d] [formerly 42f46569fa3bddc8ad944c7e7fca2829e1f3eff4 [formerly 56b44ec1f09aa670f757d04a97cdbccc3b5339d8]] [formerly 045abdc1fcaf95a2c8e5c9a8c30cbf5c358bddf0 [formerly d00f538f5c7b55bdfbe16e9a3d344685a6d4b2af] [formerly 0ca3d4c0ba036b6d902c1adb783a55cc6d393612 [formerly b326214bbbc36f48a5ac690756ad1bb9fa538cac]]] [formerly 4344f68bac5600cff2c6cc4b6a8c2356f1ddbad7 [formerly f0ae7e9d077814f4b231ffe5e6d2e9a7cee0d4d2] [formerly 813a1a286434d24f40c53d49707c3f7ea0b0a0a6 [formerly 5e348e98c8b9fd98b715e11466c389cd3b36b36d]] [formerly d1429e68ccdbe75adeffcc1b6a6a7c7e653dd299 [formerly bea65a08912a1ce768554ddd392e930af77f00fe] [formerly da1ab851944a0c6a287747af867b67440d004e57 [formerly d4d39f564f3681b3f883bd7b158d87513529b399]]]]]] Former-commit-id: 0e4c0fc5f7b19a68d880a64456538eb4f27e9ad2 [formerly fb8130714b2e93190764d9e590688f322bace637] [formerly 3d492794edae3ceaea5713874b074883b1295f83 [formerly 17a0b6934738ddb6d53eaaf656d275efbd1434ce]] [formerly b0c8ebc55b2a6f0eb2f0632bea4d85dabde4a1cc [formerly 0d73519131bc72f6cc613e4e9f1132a317c0f85a] [formerly b171b2dcd949199f625c40eee20eba2e1f56fa2a [formerly e6de695a374bac8d070d6b278e6ccaa41b28f257]]] [formerly a3f7312030818a6156d8fc3c07f5af2122c4c553 [formerly 2e8279331de4e764b3dd5efa44efddc1b312e290] [formerly 9765d5a57ddfcb6a47b37387d6ec329d49dbb8f1 [formerly 11a90a651922f555cbd2c8aa9a201a06610582c7]] [formerly f561ea4acf85731d3751816b98d5116160cf9973 [formerly e972418bab9c2cf8cb69d232f756d9cf76bb4a9b] [formerly c057b0a413c7e538506f514fddf46e8f4e5e14de [formerly 714782addeec7d30328a387987c7692cff0f7b41]]]] [formerly db10531717608173239c308df17bec70bdf2ba9b [formerly 81a1b9ee57cf59b9bd9d8a9e17186c996742c7bd] [formerly 9279abaa5973439b54ef05f4de658db605697a9d [formerly ee80e83eb858a2f2ea6b1a05e7e13b0f58d60aea]] [formerly 38a18cc2543d3bb053c7ac02034e2c727715924d [formerly e9ca3caf08b2af79915fa4876df098c04e3bfbb2] [formerly 340477e530990e0f657ff53418f0b933d9e98fba [formerly c9b004a2c38df6da8b89063eba1723dcae620d9f]]] [formerly 734509d15daab6d31cea0e422578cfe0e6880fa0 [formerly ad1cd45b341249c945b45427d580515df975e75d] [formerly 30f7c10b835ef4afc3bd7c465bfaf234524ea1b0 [formerly 710c9cc6809ad488f7d964da5139df7db8dba286]] [formerly 4a0f1409e61b8c7e05d6647c21cb1528b0544568 [formerly 7d6d940ad5684f1f6b8b96c33f4e68e63e825f49] [formerly da1ab851944a0c6a287747af867b67440d004e57]]]] Former-commit-id: 61bd2ced30dcbb60fd039bc1f09e7ac17384600f [formerly ccbc4bb634652a51f8b62fafa4f1faeb4a4baee1] [formerly 0b4b80934144b4fa07daa829910214ade40f52e9 [formerly 502825f55d16c88aad71dcfad49b15d60108be0d]] [formerly b6fd349efef160319fbed564efe3b1ff878f7674 [formerly 95572c468401fbf5bb16c72dc0022602d05cece4] [formerly d137c99d11cc86e52846a23d74898e233b9c063e [formerly f5d2a4e046a4520524b384673c1650c56225c892]]] [formerly 72e04400d64f604e72473a271be114dd171b76e6 [formerly df838a0f3951f7fc9f959fa2817f77f9abd9dcad] [formerly f61a4b36a40bc98fda5c43c17f65be47b50bbee4 [formerly bb719729f35c4e8bfb5572b0e9f8c16963106e4b]] [formerly d7983d59017477f2bbbdc14c18b3c6cc9a8599cf [formerly 485071deb0d574db7fd3918ad8d5ab9422e2c69b] [formerly fe336da38473dbefc90ff5dc7e463d704e2ad67e [formerly 4a1117cd30d141b72518b61b4492010f528bb2cb]]]] Former-commit-id: 4c31aa20027b0a09bc4ecc54401043fde9bb82e2 [formerly 368204e633021d5cc9240aec72a201c0e6ade82f] [formerly cb20d97745d17286d75ede317ea6837565b2549c [formerly 1e2bd5a92eabf0fe0b1e902ce66cce1dd0f4f386]] [formerly 6bae8da79f5b3b9025569b30c91f8a72b02b7bd6 [formerly 5e825dad6780538572f0ebff174b36fba411e1a3] [formerly 9aadaada35d3c2833a5399175ecb576dbec2dcb9 [formerly 4087666752f251a37aa027b64295374caf176812]]] Former-commit-id: 45827781fd43bffc434180c505c2c55f4544fb2f [formerly df48b013a0b96dfdcc55d6946b9db2d04155aad1] [formerly 0398938d7aef41e23eca24b00b38959ad4b83e4a [formerly cec8951ed416ae4641dbc928a2cda78caa9b7845]] Former-commit-id: a43bf0a20af7b925431591b5a5c9af1895a8d910 [formerly 2e8d60227697c9783d18e90421ee203ec41ccd35] Former-commit-id: b8cd12701effe62b7ca02079b60a2d767aabc995 --- src/axolotl | 1 + src/common-primitives | 1 + src/d3m | 1 + tods/data_processing/ContinuityValidation.py | 6 -- tods/data_processing/DuplicationValidation.py | 5 -- tods/detection_algorithm/core/LSTMOD.py | 4 +- tods/feature_analysis/BKFilter.py | 15 ++--- tods/feature_analysis/HPFilter.py | 16 ++---- tods/feature_analysis/SKTruncatedSVD.py | 82 ++++++++++++--------------- tods/feature_analysis/TRMF.py | 76 ++++++++++++------------- tods/tests/test_ContinuityValidation.py | 63 +++++++++++++++++++- tods/tests/test_DuplicationValidation.py | 48 ++++++++++++++++ 12 files changed, 197 insertions(+), 121 deletions(-) create mode 160000 src/axolotl create mode 160000 src/common-primitives create mode 160000 src/d3m diff --git a/src/axolotl b/src/axolotl new file mode 160000 index 0000000..af54e69 --- /dev/null +++ b/src/axolotl @@ -0,0 +1 @@ +Subproject commit af54e6970476a081bf0cd65990c9f56a1200d8a2 diff --git a/src/common-primitives b/src/common-primitives new file mode 160000 index 0000000..046b20d --- /dev/null +++ b/src/common-primitives @@ -0,0 +1 @@ +Subproject commit 046b20d2f6d4543dcbe18f0a1d4bcbb1f61cf518 diff --git a/src/d3m b/src/d3m new file mode 160000 index 0000000..70aeefe --- /dev/null +++ b/src/d3m @@ -0,0 +1 @@ +Subproject commit 70aeefed6b7307941581357c4b7858bb3f88e1da diff --git a/tods/data_processing/ContinuityValidation.py b/tods/data_processing/ContinuityValidation.py index 7786552..9a86560 100644 --- a/tods/data_processing/ContinuityValidation.py +++ b/tods/data_processing/ContinuityValidation.py @@ -170,9 +170,3 @@ class ContinuityValidation(transformer.TransformerPrimitiveBase[Inputs, Outputs, inputs['d3mIndex'] = list(range(inputs.shape[0])) return inputs - - def _write(self, inputs:Inputs): - """ - write inputs to current directory, only for test - """ - inputs.to_csv(str(time.time())+'.csv') diff --git a/tods/data_processing/DuplicationValidation.py b/tods/data_processing/DuplicationValidation.py index b9cdb36..2d3a85e 100644 --- a/tods/data_processing/DuplicationValidation.py +++ b/tods/data_processing/DuplicationValidation.py @@ -91,8 +91,3 @@ class DuplicationValidation(transformer.TransformerPrimitiveBase[Inputs, Outputs return inputs - def _write(self, inputs:Inputs): - """ - write inputs to current directory, only for test - """ - inputs.to_csv(str(time.time())+'.csv') diff --git a/tods/detection_algorithm/core/LSTMOD.py b/tods/detection_algorithm/core/LSTMOD.py index 636b7c1..28ced1d 100755 --- a/tods/detection_algorithm/core/LSTMOD.py +++ b/tods/detection_algorithm/core/LSTMOD.py @@ -169,7 +169,7 @@ class LSTMOutlierDetector(CollectiveBaseDetector): # print(danger_coefficient, averaged_relative_error) - else: + else: # pragma: no cover danger_coefficient = np.zeros(relative_error.shape) averaged_relative_error = np.zeros(relative_error.shape) @@ -210,7 +210,7 @@ class LSTMOutlierDetector(CollectiveBaseDetector): -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover X_train = np.asarray( [3., 4., 8., 16, 18, 13., 22., 36., 59., 128, 62, 67, 78, 100]).reshape(-1, 1) diff --git a/tods/feature_analysis/BKFilter.py b/tods/feature_analysis/BKFilter.py index c35d12c..f7633de 100644 --- a/tods/feature_analysis/BKFilter.py +++ b/tods/feature_analysis/BKFilter.py @@ -186,14 +186,14 @@ class BKFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams if len(self._training_indices) > 0: # self._clf.fit(self._training_inputs) self._fitted = True - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") - if not self._fitted: + if not self._fitted: # pragma: no cover raise PrimitiveNotFittedError("Primitive not fitted.") sk_inputs = inputs if self.hyperparams['use_semantic_types']: @@ -209,7 +209,7 @@ class BKFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams outputs.columns = self._input_column_names output_columns = [outputs] - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") @@ -217,14 +217,11 @@ class BKFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams add_index_columns=self.hyperparams['add_index_columns'], inputs=inputs, column_indices=self._training_indices, columns_list=output_columns) - - # self._write(outputs) - # self.logger.warning('produce was called3') return CallResult(outputs) @classmethod - def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): + def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): # pragma: no cover """ Select columns to fit. Args: @@ -261,7 +258,7 @@ class BKFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams # return columns_to_produce @classmethod - def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: + def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: # pragma: no cover """ Output whether a column can be processed. Args: @@ -354,8 +351,6 @@ class BKFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams return target_columns_metadata - def _write(self, inputs:Inputs): - inputs.to_csv(str(time.time())+'.csv') def _bkfilter(self, X, low, high, K): """ diff --git a/tods/feature_analysis/HPFilter.py b/tods/feature_analysis/HPFilter.py index ba26d5e..f2a5c5e 100644 --- a/tods/feature_analysis/HPFilter.py +++ b/tods/feature_analysis/HPFilter.py @@ -163,14 +163,14 @@ class HPFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams if len(self._training_indices) > 0: # self._clf.fit(self._training_inputs) self._fitted = True - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") - if not self._fitted: + if not self._fitted: # pragma: no cover raise PrimitiveNotFittedError("Primitive not fitted.") sk_inputs = inputs if self.hyperparams['use_semantic_types']: @@ -186,7 +186,7 @@ class HPFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams outputs.columns = self._input_column_names output_columns = [outputs] - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") @@ -194,14 +194,11 @@ class HPFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams add_index_columns=self.hyperparams['add_index_columns'], inputs=inputs, column_indices=self._training_indices, columns_list=output_columns) - - # self._write(outputs) - # self.logger.warning('produce was called3') return CallResult(outputs) @classmethod - def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): + def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): # pragma: no cover """ Select columns to fit. Args: @@ -238,7 +235,7 @@ class HPFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams # return columns_to_produce @classmethod - def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: + def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: # pragma: no cover """ Output whether a column can be processed. Args: @@ -331,9 +328,6 @@ class HPFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams return target_columns_metadata - def _write(self, inputs:Inputs): - inputs.to_csv(str(time.time())+'.csv') - def _hpfilter(self, X, lamb): """ Perform HPFilter diff --git a/tods/feature_analysis/SKTruncatedSVD.py b/tods/feature_analysis/SKTruncatedSVD.py index 2b6f038..01c6950 100644 --- a/tods/feature_analysis/SKTruncatedSVD.py +++ b/tods/feature_analysis/SKTruncatedSVD.py @@ -224,7 +224,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H Returns: None """ - if self._fitted: + if self._fitted: # pragma: no cover return CallResult(None) # Get cols to fit. @@ -239,7 +239,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H if len(self._training_indices) > 0: self._clf.fit(self._training_inputs) self._fitted = True - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") @@ -257,7 +257,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H # self.logger.warning(str(self.metadata.query()['name'])) - if not self._fitted: + if not self._fitted: # pragma: no cover raise PrimitiveNotFittedError("Primitive not fitted.") sk_inputs = inputs if self.hyperparams['use_semantic_types']: @@ -272,7 +272,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H if len(outputs.columns) == len(self._input_column_names): outputs.columns = self._input_column_names output_columns = [outputs] - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") @@ -286,7 +286,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H return CallResult(outputs) - def get_params(self) -> Params: + def get_params(self) -> Params: # pragma: no cover """ Return parameters. Args: @@ -320,7 +320,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H target_columns_metadata_=self._target_columns_metadata ) - def set_params(self, *, params: Params) -> None: + def set_params(self, *, params: Params) -> None: # pragma: no cover """ Set parameters for SKTruncatedSVD. Args: @@ -351,7 +351,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H @classmethod - def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): + def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): # pragma: no cover """ Select columns to fit. Args: @@ -377,7 +377,7 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H # return columns_to_produce @classmethod - def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: + def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: # pragma: no cover """ Output whether a column can be processed. Args: @@ -408,35 +408,35 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H return False - @classmethod - def _get_target_columns_metadata(cls, outputs_metadata: metadata_base.DataMetadata, hyperparams) -> List[OrderedDict]: - """ - Output metadata of selected columns. - Args: - outputs_metadata: metadata_base.DataMetadata - hyperparams: d3m.metadata.hyperparams.Hyperparams - - Returns: - d3m.metadata.base.DataMetadata - """ - outputs_length = outputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] - - target_columns_metadata: List[OrderedDict] = [] - for column_index in range(outputs_length): - column_metadata = OrderedDict(outputs_metadata.query_column(column_index)) - - # Update semantic types and prepare it for predicted targets. - semantic_types = set(column_metadata.get('semantic_types', [])) - semantic_types_to_remove = set([]) - add_semantic_types = [] - add_semantic_types.add(hyperparams["return_semantic_type"]) - semantic_types = semantic_types - semantic_types_to_remove - semantic_types = semantic_types.union(add_semantic_types) - column_metadata['semantic_types'] = list(semantic_types) - - target_columns_metadata.append(column_metadata) - - return target_columns_metadata + # @classmethod + # def _get_target_columns_metadata(cls, outputs_metadata: metadata_base.DataMetadata, hyperparams) -> List[OrderedDict]: + # """ + # Output metadata of selected columns. + # Args: + # outputs_metadata: metadata_base.DataMetadata + # hyperparams: d3m.metadata.hyperparams.Hyperparams + + # Returns: + # d3m.metadata.base.DataMetadata + # """ + # outputs_length = outputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + # target_columns_metadata: List[OrderedDict] = [] + # for column_index in range(outputs_length): + # column_metadata = OrderedDict(outputs_metadata.query_column(column_index)) + + # # Update semantic types and prepare it for predicted targets. + # semantic_types = set(column_metadata.get('semantic_types', [])) + # semantic_types_to_remove = set([]) + # add_semantic_types = [] + # add_semantic_types.add(hyperparams["return_semantic_type"]) + # semantic_types = semantic_types - semantic_types_to_remove + # semantic_types = semantic_types.union(add_semantic_types) + # column_metadata['semantic_types'] = list(semantic_types) + + # target_columns_metadata.append(column_metadata) + + # return target_columns_metadata @classmethod def _update_predictions_metadata(cls, inputs_metadata: metadata_base.DataMetadata, outputs: Optional[Outputs], @@ -500,11 +500,3 @@ class SKTruncatedSVD(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, Params, H return target_columns_metadata - def _write(self, inputs:Inputs): - """ - write inputs to current directory, only for test - """ - inputs.to_csv(str(time.time())+'.csv') - - -# SKTruncatedSVD.__doc__ = TruncatedSVD.__doc__ diff --git a/tods/feature_analysis/TRMF.py b/tods/feature_analysis/TRMF.py index 3b21fa1..fcbdab9 100644 --- a/tods/feature_analysis/TRMF.py +++ b/tods/feature_analysis/TRMF.py @@ -276,14 +276,14 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): if len(self._training_indices) > 0: self._clf.fit(self._training_inputs) self._fitted = True - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") - if not self._fitted: + if not self._fitted: # pragma: no cover raise PrimitiveNotFittedError("Primitive not fitted.") sk_inputs = inputs @@ -301,7 +301,7 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): if len(outputs.columns) == len(self._input_column_names): outputs.columns = self._input_column_names output_columns = [outputs] - else: + else: # pragma: no cover if self.hyperparams['error_on_no_input']: raise RuntimeError("No input columns were selected") self.logger.warn("No input columns were selected") @@ -316,7 +316,7 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): @classmethod - def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): + def _get_columns_to_fit(cls, inputs: Inputs, hyperparams: Hyperparams): # pragma: no cover """ Select columns to fit. Args: @@ -342,7 +342,7 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): # return columns_to_produce @classmethod - def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: + def _can_produce_column(cls, inputs_metadata: metadata_base.DataMetadata, column_index: int, hyperparams: Hyperparams) -> bool: # pragma: no cover """ Output whether a column can be processed. Args: @@ -373,35 +373,35 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): return False - @classmethod - def _get_target_columns_metadata(cls, outputs_metadata: metadata_base.DataMetadata, hyperparams) -> List[OrderedDict]: - """ - Output metadata of selected columns. - Args: - outputs_metadata: metadata_base.DataMetadata - hyperparams: d3m.metadata.hyperparams.Hyperparams - - Returns: - d3m.metadata.base.DataMetadata - """ - outputs_length = outputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] - - target_columns_metadata: List[OrderedDict] = [] - for column_index in range(outputs_length): - column_metadata = OrderedDict(outputs_metadata.query_column(column_index)) - - # Update semantic types and prepare it for predicted targets. - semantic_types = set(column_metadata.get('semantic_types', [])) - semantic_types_to_remove = set([]) - add_semantic_types = [] - add_semantic_types.add(hyperparams["return_semantic_type"]) - semantic_types = semantic_types - semantic_types_to_remove - semantic_types = semantic_types.union(add_semantic_types) - column_metadata['semantic_types'] = list(semantic_types) - - target_columns_metadata.append(column_metadata) - - return target_columns_metadata + # @classmethod + # def _get_target_columns_metadata(cls, outputs_metadata: metadata_base.DataMetadata, hyperparams) -> List[OrderedDict]: + # """ + # Output metadata of selected columns. + # Args: + # outputs_metadata: metadata_base.DataMetadata + # hyperparams: d3m.metadata.hyperparams.Hyperparams + + # Returns: + # d3m.metadata.base.DataMetadata + # """ + # outputs_length = outputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + # target_columns_metadata: List[OrderedDict] = [] + # for column_index in range(outputs_length): + # column_metadata = OrderedDict(outputs_metadata.query_column(column_index)) + + # # Update semantic types and prepare it for predicted targets. + # semantic_types = set(column_metadata.get('semantic_types', [])) + # semantic_types_to_remove = set([]) + # add_semantic_types = [] + # add_semantic_types.add(hyperparams["return_semantic_type"]) + # semantic_types = semantic_types - semantic_types_to_remove + # semantic_types = semantic_types.union(add_semantic_types) + # column_metadata['semantic_types'] = list(semantic_types) + + # target_columns_metadata.append(column_metadata) + + # return target_columns_metadata @classmethod def _update_predictions_metadata(cls, inputs_metadata: metadata_base.DataMetadata, outputs: Optional[Outputs], @@ -465,12 +465,6 @@ class TRMF(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): return target_columns_metadata - def _write(self, inputs:Inputs): - """ - write inputs to current directory, only for test - """ - inputs.to_csv(str(time.time())+'.csv') - """ Temporal Regularized Matrix Factorization @@ -564,7 +558,7 @@ class trmf: return np.dot(self.F, X_preds) - def _predict_X(self, h): + def _predict_X(self, h): # pragma: no cover """Predict X h timepoints ahead. Evaluates matrix X with the help of matrix W. diff --git a/tods/tests/test_ContinuityValidation.py b/tods/tests/test_ContinuityValidation.py index 185bd4c..9c69f8e 100644 --- a/tods/tests/test_ContinuityValidation.py +++ b/tods/tests/test_ContinuityValidation.py @@ -59,7 +59,7 @@ class ContinuityValidationTest(unittest.TestCase): hyperparams_class = ContinuityValidation.ContinuityValidation.metadata.get_hyperparams() primitive = ContinuityValidation.ContinuityValidation(hyperparams=hyperparams_class.defaults()) new_main = primitive.produce(inputs=main).value - # print(new_main) + expected_output = container.DataFrame({'d3mIndex': [0, 1, 2, 3], 'timestamp': [1., 2., 3., 4.], @@ -124,6 +124,67 @@ class ContinuityValidationTest(unittest.TestCase): self._test_continuity(new_main) + hyperparams = hyperparams_class.defaults() + hyperparams = hyperparams.replace({'continuity_option': 'ablation'}) + primitive2 = ContinuityValidation.ContinuityValidation(hyperparams=hyperparams) + new_main2 = primitive2.produce(inputs=main).value + print(new_main2) + + self.assertEqual(utils.to_json_structure(new_main2.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 2, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 5, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': { + 'name': 'd3mIndex', + 'structural_type': 'numpy.int64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': { + 'name': 'timestamp', + 'structural_type': 'numpy.float64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': { + 'name': 'a', + 'structural_type': 'numpy.float64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 3], + 'metadata': { + 'name': 'b', + 'structural_type': 'numpy.float64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 4], + 'metadata': { + 'name': 'ground_truth', + 'structural_type': 'numpy.int64', + }, + }]) + + def _test_continuity(self, data_value): tmp_col = data_value['timestamp'] diff --git a/tods/tests/test_DuplicationValidation.py b/tods/tests/test_DuplicationValidation.py index 6c85cfb..6e34d1f 100644 --- a/tods/tests/test_DuplicationValidation.py +++ b/tods/tests/test_DuplicationValidation.py @@ -102,6 +102,54 @@ class DuplicationValidationTest(unittest.TestCase): self._test_drop_duplication(new_main) + hyperparams = hyperparams_class.defaults() + hyperparams = hyperparams.replace({'keep_option': 'average'}) + primitive2 = DuplicationValidation.DuplicationValidation(hyperparams=hyperparams) + new_main2 = primitive2.produce(inputs=main).value + print(new_main2) + + self.assertEqual(utils.to_json_structure(new_main.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 2, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 3, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': { + 'name': 'timestamp', + 'structural_type': 'numpy.float64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': { + 'name': 'a', + 'structural_type': 'numpy.float64', + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': { + 'name': 'b', + 'structural_type': 'numpy.float64', + }, + }]) + def _test_drop_duplication(self, data_value): self.assertEqual(True in list(data_value.duplicated('timestamp')), False)