diff --git a/1614317214.141085.csv b/1614317214.141085.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614317214.141085.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/1614317813.009414.csv b/1614317813.009414.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614317813.009414.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/1614320539.096673.csv b/1614320539.096673.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614320539.096673.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/1614322080.566302.csv b/1614322080.566302.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614322080.566302.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/1614435080.68491.csv b/1614435080.68491.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614435080.68491.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/1614437760.588848.csv b/1614437760.588848.csv new file mode 100644 index 0000000..80343ab --- /dev/null +++ b/1614437760.588848.csv @@ -0,0 +1,4 @@ +output_0,output_1,output_2 +1.0,2.0,3.0 +2.0,3.0,4.0 +3.0,4.0,5.0 diff --git a/tods/common/CSVReader.py b/tods/common/CSVReader.py index 662d62d..8c1fad6 100644 --- a/tods/common/CSVReader.py +++ b/tods/common/CSVReader.py @@ -11,7 +11,7 @@ from d3m.base import primitives -class CSVReaderPrimitive(primitives.FileReaderPrimitiveBase): +class CSVReaderPrimitive(primitives.FileReaderPrimitiveBase): # pragma: no cover """ A primitive which reads columns referencing CSV files. diff --git a/tods/common/Denormalize.py b/tods/common/Denormalize.py index 2cadae6..b5f708d 100644 --- a/tods/common/Denormalize.py +++ b/tods/common/Denormalize.py @@ -50,7 +50,7 @@ class Hyperparams(hyperparams.Hyperparams): # once to the second table, and then just do the join with already joined second table. # Not sure how to behave in "recursive == False" case then. # TODO: Add a test where main table has a foreign key twice to same table (for example, person 1 and person 2 to table of persons). -class DenormalizePrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): +class DenormalizePrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): # pragma: no cover """ A primitive which converts a Dataset with multiple tabular resources into a Dataset with only one tabular resource, based on known relations between tabular resources. Any resource which can be joined is joined (thus the resource @@ -517,28 +517,28 @@ class DenormalizePrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, del graph[resource_id] -if __name__ == '__main__': - import logging - import pprint - import sys +# if __name__ == '__main__': +# import logging +# import pprint +# import sys - logging.basicConfig() +# logging.basicConfig() - for dataset_file_path in sys.argv[1:]: - try: - dataset = container.Dataset.load('file://{dataset_doc_path}'.format(dataset_doc_path=os.path.abspath(dataset_file_path))) - except Exception as error: - raise Exception("Unable to load dataset: {dataset_doc_path}".format(dataset_doc_path=dataset_file_path)) from error +# for dataset_file_path in sys.argv[1:]: +# try: +# dataset = container.Dataset.load('file://{dataset_doc_path}'.format(dataset_doc_path=os.path.abspath(dataset_file_path))) +# except Exception as error: +# raise Exception("Unable to load dataset: {dataset_doc_path}".format(dataset_doc_path=dataset_file_path)) from error - primitive = DenormalizePrimitive(hyperparams=Hyperparams.defaults().replace({ - 'recursive': True, - 'discard_not_joined_tabular_resources': False, - })) +# primitive = DenormalizePrimitive(hyperparams=Hyperparams.defaults().replace({ +# 'recursive': True, +# 'discard_not_joined_tabular_resources': False, +# })) - try: - denormalized_dataset = primitive.produce(inputs=dataset).value +# try: +# denormalized_dataset = primitive.produce(inputs=dataset).value - pprint.pprint(denormalized_dataset) - denormalized_dataset.metadata.pretty_print() - except Exception as error: - raise Exception("Unable to denormalize dataset: {dataset_doc_path}".format(dataset_doc_path=dataset_file_path)) from error +# pprint.pprint(denormalized_dataset) +# denormalized_dataset.metadata.pretty_print() +# except Exception as error: +# raise Exception("Unable to denormalize dataset: {dataset_doc_path}".format(dataset_doc_path=dataset_file_path)) from error diff --git a/tods/data_processing/ColumnParser.py b/tods/data_processing/ColumnParser.py index 40e9b2e..9a27b5b 100644 --- a/tods/data_processing/ColumnParser.py +++ b/tods/data_processing/ColumnParser.py @@ -134,7 +134,7 @@ class ColumnParserPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs return base.CallResult(outputs) - def _can_use_column(self, inputs_metadata: metadata_base.DataMetadata, column_index: int) -> bool: + def _can_use_column(self, inputs_metadata: metadata_base.DataMetadata, column_index: int) -> bool: # pragma: no cover column_metadata = inputs_metadata.query((metadata_base.ALL_ELEMENTS, column_index)) # We produce only on columns which have not yet been parsed (are strings). @@ -156,7 +156,7 @@ class ColumnParserPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs return False - def _produce_columns(self, inputs: Inputs) -> typing.Tuple[typing.List[int], typing.List[Outputs]]: + def _produce_columns(self, inputs: Inputs) -> typing.Tuple[typing.List[int], typing.List[Outputs]]: # pragma: no cover # The logic of parsing values tries to mirror also the logic of detecting # values in "SimpleProfilerPrimitive". One should keep them in sync. diff --git a/tods/reinforcement/RuleBasedFilter.py b/tods/reinforcement/RuleBasedFilter.py index f87ed53..fe2496e 100644 --- a/tods/reinforcement/RuleBasedFilter.py +++ b/tods/reinforcement/RuleBasedFilter.py @@ -6,7 +6,6 @@ import os import sklearn import numpy import typing -import time from d3m import container from d3m.primitive_interfaces import base, transformer @@ -22,8 +21,6 @@ from d3m.primitive_interfaces.base import CallResult, DockerContainer import os.path - -import time import re @@ -31,7 +28,7 @@ Inputs = container.DataFrame Outputs = container.DataFrame -class Hyperparams(hyperparams.Hyperparams): +class Hyperparams(hyperparams.Hyperparams): # pragma: no cover # Tuning rule = hyperparams.Hyperparameter[str]( default='', @@ -81,7 +78,7 @@ class Hyperparams(hyperparams.Hyperparams): ) -class RuleBasedFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): +class RuleBasedFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): # pragma: no cover """ Filter the selected columns according to the rule. @@ -143,8 +140,9 @@ class RuleBasedFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hype operated_col = [int(x.strip('#')) for x in re.findall(r'#\d*#', self.hyperparams['rule'])] + + if set(operated_col) != set(self._training_indices): - # print(operated_col, self._training_indices) raise RuntimeError("Column numbers in 'rule' and 'use_columns' are not matched.") @@ -186,8 +184,6 @@ class RuleBasedFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hype inputs=inputs, column_indices=self._training_indices, columns_list=output_columns) - # self._write(outputs) - # self.logger.warning('produce was called3') return CallResult(outputs) @@ -223,7 +219,6 @@ class RuleBasedFilter(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hype # else: use_columns=hyperparams['use_columns'] exclude_columns=hyperparams['exclude_columns'] - columns_to_produce, columns_not_to_produce = base_utils.get_columns_to_use(inputs_metadata, use_columns=use_columns, exclude_columns=exclude_columns, can_use_column=can_produce_column) return inputs.iloc[:, columns_to_produce], columns_to_produce diff --git a/tods/sk_interface/data_ensemble/Ensemble_skinterface.py b/tods/sk_interface/data_ensemble/Ensemble_skinterface.py index 5e3a0ce..a0ed7eb 100644 --- a/tods/sk_interface/data_ensemble/Ensemble_skinterface.py +++ b/tods/sk_interface/data_ensemble/Ensemble_skinterface.py @@ -3,7 +3,7 @@ from d3m import container from tods.detection_algorithm.Ensemble import EnsemblePrimitive from ..base import get_default_hyperparameter -class EnsembleSKI: +class EnsembleSKI: # pragma: no cover def __init__(self, **hyperparameter): hyperparams = get_default_hyperparameter(EnsemblePrimitive, hyperparameter) diff --git a/tods/tests/common/test_csv_reader.py b/tods/tests/common/test_csv_reader.py index 60d8ae0..07b8fb1 100644 --- a/tods/tests/common/test_csv_reader.py +++ b/tods/tests/common/test_csv_reader.py @@ -9,7 +9,7 @@ from tods.data_processing import DatasetToDataframe class CSVReaderPrimitiveTestCase(unittest.TestCase): - def _get_yahoo_dataset(self): + def _get_yahoo_dataset(self): # pragma: no cover dataset_doc_path = os.path.abspath(os.path.join(os.path.dirname(__file__),'..', '..', '..', 'datasets', 'anomaly','yahoo_sub_5','TRAIN','dataset_TRAIN', 'datasetDoc.json')) dataset = container.Dataset.load('file://{dataset_doc_path}'.format(dataset_doc_path=dataset_doc_path)) diff --git a/tods/tests/common/test_denormalize.py b/tods/tests/common/test_denormalize.py index 5b440cc..2df6448 100644 --- a/tods/tests/common/test_denormalize.py +++ b/tods/tests/common/test_denormalize.py @@ -46,7 +46,7 @@ class DenormalizePrimitiveTestCase(unittest.TestCase): #self.assertEqual(dataset.metadata.to_internal_json_structure(), dataset_metadata_before) - def _test_discard_metadata(self, metadata, dataset_doc_path): + def _test_discard_metadata(self, metadata, dataset_doc_path): # pragma: no cover self.maxDiff = None self.assertEqual(test_utils.convert_through_json(metadata.query(())), { @@ -197,7 +197,7 @@ class DenormalizePrimitiveTestCase(unittest.TestCase): #self.assertEqual(dataset.metadata.to_internal_json_structure(), dataset_metadata_before) - def _test_recursive_metadata(self, metadata, dataset_doc_path): + def _test_recursive_metadata(self, metadata, dataset_doc_path): # pragma: no cover self.maxDiff = None self.assertEqual(test_utils.convert_through_json(metadata.query(())), { @@ -344,7 +344,7 @@ class DenormalizePrimitiveTestCase(unittest.TestCase): #self.assertEqual(dataset.metadata.to_internal_json_structure(), dataset_metadata_before) - def _test_row_order_metadata(self, metadata, dataset_doc_path): + def _test_row_order_metadata(self, metadata, dataset_doc_path): # pragma: no cover self.maxDiff = None self.assertEqual(test_utils.convert_through_json(metadata.query(())), {