From ced19bb59d3a70da8fb71b4ba03a9af4aa2976d4 Mon Sep 17 00:00:00 2001 From: YileAllenChen1 Date: Mon, 28 Sep 2020 10:00:39 -0500 Subject: [PATCH] add missing files Former-commit-id: 3f7cfc37e86a90fca586dad50b3775107aa83973 [formerly b74d2356a8b3831158e558a2fdfeca84a6e4b1a0] [formerly 3c458d75c2c43fd0d1da6f9c684ebbd9a2c1d296 [formerly a2a165609addab7cdb37d534e4755e8bfb0d08fe]] [formerly 2e3b925ec76c4e13d38a8a917f4063509c4b7390 [formerly e31b06239fefc32b0ffdbeaeadfc7689d007c67b] [formerly 31ea5772a143ecfa7d7f36280fc0735f1c3bd12c [formerly 0a384a898453a49398e6dd2aa9c9280682e5d9a8]]] [formerly 2bf9721c550465e34385174d0fb187f86d68f523 [formerly 493f4c3014dd08af979c70bb271c4b50efde783d] [formerly f1974f2891657daff13ac3a433ff2a3377d7d4a3 [formerly 81227824d76bd2030bfa6b6074a14f56b101f2dc]] [formerly f3db06651b91d4de1a9fadfd82f6a397c56e92b7 [formerly 8d78b06e9e92f76c8ee5335b34d6532ace60b33d] [formerly f907ef71f508918a6674244cdd8735f28a7b8eaa [formerly cf532ec08377986509f9354a3765ce730a41daec]]]] [formerly 946bb682411b6fa0f639bd0ac982e5b6575cd29f [formerly 96ba9ceb7d249da2a61e929a33583cb4df5558ec] [formerly 180343f152b5852b170ab83fc55ee94451165f57 [formerly 9fbd0ca43af4b4af3f0e80b73e0feaad5ad7af17]] [formerly cb24f146f1d07d1ce68a04276e181dc2b894ebcc [formerly 414b396fc2a4b8f9dec8639801cc57e8f6098004] [formerly d304cadd7fc9a9ccbb2284c2c0f3e53483307973 [formerly 329acba2f7dc64202feaee19e323ebe42e7cc7b8]]] [formerly 44cbbbdeb6bc28f3ebb71ea6f400d6ebeb5b86ad [formerly 3b1d52222ce5996dc1f3194ecfa71e8ab6742a24] [formerly 795250861eeaf590e52d176ac057410d2a8c486d [formerly d0ab4d27cb87c5456fcfb5e7515dd4b456859876]] [formerly 99abf724fdac301b8f076ad0348dc3106c10e980 [formerly 5f4009353953b808fcccfaf8fba767ccf6f67a4a] [formerly 3fb7bbd40a056f37a06adda86df3b29b1d4c332d [formerly 908f6374bf0e411ebde61a54e69d598ef6be497b]]]]] [formerly 156900ecff3b051b0eb4b806984f986612409b27 [formerly f7b387c445d9b8b0dd1de85e29cc33a8b3d19f4b] [formerly 92c76931e34c984b6dd01b64103753defbb647de [formerly c7731b7d462243ceeaa70a7bb161d90a7bccfd37]] [formerly 339e1c29e87dca6a4eadde6b825523934f874d09 [formerly 20a8356f5a1d2ff69283016fe2081dbdc724d259] [formerly 5b71a77b6b3371993d0fba884f9f28ee0f6c4ae3 [formerly 6a57734672f5c426feab2be8ef463e13d727228f]]] [formerly 6717e7f53e47131a49668c89fc30ffb2abdb6a7a [formerly a432c417ff0316c84bff3d5382020504794eda2c] [formerly 1fe8593ba4cf862ed900b9d95d0d93fa8f707a7a [formerly 568256d5721346879966dec0655efa19b9720b59]] [formerly d841eea05b3142543c90040e3e8f86aafddc47c8 [formerly 58c80baa9471d620e1e86051c9bf2808e0f69c2a] [formerly b2e4d6bc92bced9c3a718391f103b8d07075c14a [formerly 809a919c22db495675b51935025080e40e78017e]]]] [formerly 7d24f921fcc07d1a2c1a70f5e3701103bb1c66c9 [formerly 5f164935685a1f97845602bd4f683cb19b48a7be] [formerly 32434bfce54404fe92191ec5c36773011fe17412 [formerly 773ffcf35022c051bb0a670a1028a44184b35655]] [formerly 174627e1149409ec7cc1eefd5bcbff7b6e3ff644 [formerly cf159d2675ebf5818649078e8511324cd3b68fb1] [formerly 09c70e1cc966bc86419fe5b2d70a3c0dbe95be90 [formerly 2b9c213aad00bba1ccdb9cbdeb44ad5f3db20e86]]] [formerly b6c669ab04fda2b3a8d748f01603dcc7de050ff1 [formerly 7c0b650d34f16050fc66ebff942a4d3d1f74581d] [formerly 2e210feb12122a4f044364b8c02c67acce600091 [formerly 7143d46ab403e55af33fbeb5825e0a5dfb78be0f]] [formerly bbf4c5f3aad57a7683ef2b8cd1460492428de0f3 [formerly b6370d4615e618b16bf953abd548315c1d9d728d] [formerly b7147eadb4a7ffde65feb1dc738580bd6465a0b1 [formerly 501035efa98214aff76182ac5b66a8cd991e4552]]]]]] Former-commit-id: 49012012f682c882f9bf1b8e3eeddc9f418622ba [formerly b9bb9e149905c00ebfefa2e873c6fb4b438fc7a0] [formerly 4a6c6000484d9657eea2140dc79406ac1c27c647 [formerly e7e139db3f59ba05b8c684c290d16483de61e03b]] [formerly 78b2a6c046be1c15c316d0261bf1b81e9620fc39 [formerly f641bd1128a548af799dc835e7c6914704b5570e] [formerly 1c7a5915045875f848a0bbcd8d73ca17e106b477 [formerly 67689a8d9c9bc19977ded936c050ffe5ca238f37]]] [formerly 797c03e482853fcb5944f4fafbafac964ab5dfda [formerly 2bbff2c4dbb2f6bbe1e67a3dbedac24f95a38a97] [formerly 9ab8c745ee23cc0951c95cdb022d1b34df433443 [formerly dc5e07cb3f3e11077244883e756f3aa35af9fa06]] [formerly 12fcc4027ff8323b21eed428369ea6eb6a8cf004 [formerly ec3f1d9078d85028bc405ea07119d52f9c10c098] [formerly b3d4bcd0a5c57dd861d6d0fc57d8987268e165f6 [formerly d73228793c605757f167dc66119b2a0d5a96f3d7]]]] [formerly 912e6c46ec3ac5741f7f9b8f4305fc37b58925a2 [formerly 52270242af227a84e59847efb09cbd10dedeaccc] [formerly c8b312f42ab3fac8a952a52ad54ca22e42a78922 [formerly d40c96306e8c8774f134a42c10f47de50860e563]] [formerly 84e9e155a4f43f77dd72858c84a937d112d51101 [formerly 8a8acd041a7c1e962b2eb4a8e08634a72d6a55f6] [formerly 6115e6e49f6b305d1956a3c8ac17599ca14523b8 [formerly 643cff68dfa69a5dd987b113546b92f2faac424a]]] [formerly ee555be73d572fd8dd0beeb2f17030012ef5a262 [formerly 5a214858f40eee04ac469f516b913191413bec06] [formerly 5cc2dade41ff380fdccec9deda1ca24d9de410f8 [formerly c792c83235b32acd37403cc0f441a8bbe8d25267]] [formerly 5932e3f80773785caa98c07d92d20e7a6fc23890 [formerly 3390f186bc726c8cc931f130c3fe05339aeda5eb] [formerly b7147eadb4a7ffde65feb1dc738580bd6465a0b1]]]] Former-commit-id: 71accf5c5f5a652f6bd88a363c9d526307a1651b [formerly 52bc81705bb514dc7af6ab5fa2cd310338c3af18] [formerly 367a0893b3cfe7dc17418856a4ab300909c01018 [formerly e5f4ba0713aefb453794fae21a986d951746b3d1]] [formerly e7f38d40738a0fbb89e96fe675044a73469fa5da [formerly bea0027d207e3d84a0336998c485c59e03df22ac] [formerly a0d9e910f17b6ca8a846ace80a7f8bbe55b271be [formerly e936a0d13e5ef8b7dd50c8201f0eb1b42e542275]]] [formerly 2eb4bfb3fc4c314e8fea7b566fbdbc2a2ce31ff0 [formerly 5769769ddc428fcb5dedae1b9ad9f7d0eab8e70a] [formerly 61e28b2083a5b9b7e4f087c88724503cc1bd97f4 [formerly 4d1b5b5d3948319ffa35bad079d12e8591c1f3fb]] [formerly d939d3e623dac9494f10e0301cb23c4c6b5760b8 [formerly 091ffd784f0b59d095afd0a4c390e360f26a44bd] [formerly bf0416419e96e70806534cebff15d7c06d26d4f5 [formerly 0f38b48d921d562a6077f8919b4be06659412d7f]]]] Former-commit-id: 71dff5394cb27e499900616d2debdf57124d25c3 [formerly faebf231de0c2faee46721ccd42f11d19a866355] [formerly 2f13c6357f79781fb59586b03403f18cd92ddc88 [formerly 3014c51839e540cedcc9f502aead80821f077138]] [formerly d8ff0b66fc9c2a6774837c1802636953f517a1a2 [formerly c0403e640146c8c48ff7ecf7bf718d255c027372] [formerly 4f37326f4fbcf44455eb58636187f91f1aa6a3ed [formerly 035a2e6ff3ccfa6d041fe6fae28208b320b9d6f4]]] Former-commit-id: 1e2d5f638acdc448d6f80ac04a410aae099c3bff [formerly e49643e7de973ac446e6604efc5155c201c1eb33] [formerly 3fe5eb8c6bf69f627b6c5eb4d6434420dfaa12b9 [formerly a5428797f097d75c4d1b25125c1fe9b0c2d3248b]] Former-commit-id: 887b61afe5ca8136a10e17fff95c58778014aaa4 [formerly dba4aebac6073e9abef46844a24b03ddae54a545] Former-commit-id: 66f8f84a577bbe6dffa51169cbef55a7e99fa6a5 --- examples/build_Autocorrelation_pipeline.py | 70 +++++++++++++++ examples/build_DeepLog_pipeline.py | 70 +++++++++++++++ examples/build_MatrixProfile_pipeline.py | 70 +++++++++++++++ examples/build_SOD_pipeline.py | 70 +++++++++++++++ tods/tests/test_Autocorrelation.py | 134 +++++++++++++++++++++++++++++ tods/tests/test_TimeIntervalTransform.py | 121 ++++++++++++++++++++++++++ 6 files changed, 535 insertions(+) create mode 100644 examples/build_Autocorrelation_pipeline.py create mode 100644 examples/build_DeepLog_pipeline.py create mode 100644 examples/build_MatrixProfile_pipeline.py create mode 100644 examples/build_SOD_pipeline.py create mode 100644 tods/tests/test_Autocorrelation.py create mode 100644 tods/tests/test_TimeIntervalTransform.py diff --git a/examples/build_Autocorrelation_pipeline.py b/examples/build_Autocorrelation_pipeline.py new file mode 100644 index 0000000..4242e73 --- /dev/null +++ b/examples/build_Autocorrelation_pipeline.py @@ -0,0 +1,70 @@ +from d3m import index +from d3m.metadata.base import ArgumentType +from d3m.metadata.pipeline import Pipeline, PrimitiveStep + +# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest +# extract_columns_by_semantic_types(targets) -> ^ + +# Creating pipeline +pipeline_description = Pipeline() +pipeline_description.add_input(name='inputs') + +# Step 0: dataset_to_dataframe +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_output('produce') +pipeline_description.add_step(step_0) + +# Step 1: column_parser +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_output('produce') +pipeline_description.add_step(step_1) + +# Step 2: extract_columns_by_semantic_types(attributes) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_output('produce') +step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/Attribute']) +pipeline_description.add_step(step_2) + +# Step 3: extract_columns_by_semantic_types(targets) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_3.add_output('produce') +step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) +pipeline_description.add_step(step_3) + +attributes = 'steps.2.produce' +targets = 'steps.3.produce' + +# Step 4: processing +step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler')) +step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) +step_4.add_output('produce') +pipeline_description.add_step(step_4) + +# Step 5: algorithm` +step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.auto_correlation')) +step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') +step_5.add_output('produce') +pipeline_description.add_step(step_5) + +# Step 6: Predictions +step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.construct_predictions.Common')) +step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') +step_6.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_6.add_output('produce') +pipeline_description.add_step(step_6) + +# Final Output +pipeline_description.add_output(name='output predictions', data_reference='steps.6.produce') + +# Output to json +data = pipeline_description.to_json() +with open('example_pipeline.json', 'w') as f: + f.write(data) + print(data) + diff --git a/examples/build_DeepLog_pipeline.py b/examples/build_DeepLog_pipeline.py new file mode 100644 index 0000000..21fd586 --- /dev/null +++ b/examples/build_DeepLog_pipeline.py @@ -0,0 +1,70 @@ +from d3m import index +from d3m.metadata.base import ArgumentType +from d3m.metadata.pipeline import Pipeline, PrimitiveStep + +# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest +# extract_columns_by_semantic_types(targets) -> ^ + +# Creating pipeline +pipeline_description = Pipeline() +pipeline_description.add_input(name='inputs') + +# Step 0: dataset_to_dataframe +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_output('produce') +pipeline_description.add_step(step_0) + +# Step 1: column_parser +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_output('produce') +pipeline_description.add_step(step_1) + +# Step 2: extract_columns_by_semantic_types(attributes) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_output('produce') +step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/Attribute']) +pipeline_description.add_step(step_2) + +# Step 3: extract_columns_by_semantic_types(targets) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_3.add_output('produce') +step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) +pipeline_description.add_step(step_3) + +attributes = 'steps.2.produce' +targets = 'steps.3.produce' + +# Step 4: processing +step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler')) +step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) +step_4.add_output('produce') +pipeline_description.add_step(step_4) + +# Step 5: algorithm` +step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.deeplog')) +step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') +step_5.add_output('produce') +pipeline_description.add_step(step_5) + +# Step 6: Predictions +step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.construct_predictions.Common')) +step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') +step_6.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_6.add_output('produce') +pipeline_description.add_step(step_6) + +# Final Output +pipeline_description.add_output(name='output predictions', data_reference='steps.6.produce') + +# Output to json +data = pipeline_description.to_json() +with open('example_pipeline.json', 'w') as f: + f.write(data) + print(data) + diff --git a/examples/build_MatrixProfile_pipeline.py b/examples/build_MatrixProfile_pipeline.py new file mode 100644 index 0000000..3d1e66c --- /dev/null +++ b/examples/build_MatrixProfile_pipeline.py @@ -0,0 +1,70 @@ +from d3m import index +from d3m.metadata.base import ArgumentType +from d3m.metadata.pipeline import Pipeline, PrimitiveStep + +# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest +# extract_columns_by_semantic_types(targets) -> ^ + +# Creating pipeline +pipeline_description = Pipeline() +pipeline_description.add_input(name='inputs') + +# Step 0: dataset_to_dataframe +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_output('produce') +pipeline_description.add_step(step_0) + +# Step 1: column_parser +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_output('produce') +pipeline_description.add_step(step_1) + +# Step 2: extract_columns_by_semantic_types(attributes) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_output('produce') +step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/Attribute']) +pipeline_description.add_step(step_2) + +# Step 3: extract_columns_by_semantic_types(targets) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_3.add_output('produce') +step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) +pipeline_description.add_step(step_3) + +attributes = 'steps.2.produce' +targets = 'steps.3.produce' + +# Step 4: processing +step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler')) +step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) +step_4.add_output('produce') +pipeline_description.add_step(step_4) + +# Step 5: algorithm` +step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.matrix_profile')) +step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') +step_5.add_output('produce') +pipeline_description.add_step(step_5) + +# Step 6: Predictions +step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.construct_predictions.Common')) +step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') +step_6.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_6.add_output('produce') +pipeline_description.add_step(step_6) + +# Final Output +pipeline_description.add_output(name='output predictions', data_reference='steps.6.produce') + +# Output to json +data = pipeline_description.to_json() +with open('example_pipeline.json', 'w') as f: + f.write(data) + print(data) + diff --git a/examples/build_SOD_pipeline.py b/examples/build_SOD_pipeline.py new file mode 100644 index 0000000..9e92d0b --- /dev/null +++ b/examples/build_SOD_pipeline.py @@ -0,0 +1,70 @@ +from d3m import index +from d3m.metadata.base import ArgumentType +from d3m.metadata.pipeline import Pipeline, PrimitiveStep + +# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest +# extract_columns_by_semantic_types(targets) -> ^ + +# Creating pipeline +pipeline_description = Pipeline() +pipeline_description.add_input(name='inputs') + +# Step 0: dataset_to_dataframe +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_output('produce') +pipeline_description.add_step(step_0) + +# Step 1: column_parser +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_output('produce') +pipeline_description.add_step(step_1) + +# Step 2: extract_columns_by_semantic_types(attributes) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_output('produce') +step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/Attribute']) +pipeline_description.add_step(step_2) + +# Step 3: extract_columns_by_semantic_types(targets) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_3.add_output('produce') +step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, + data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) +pipeline_description.add_step(step_3) + +attributes = 'steps.2.produce' +targets = 'steps.3.produce' + +# Step 4: processing +step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler')) +step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) +step_4.add_output('produce') +pipeline_description.add_step(step_4) + +# Step 5: algorithm` +step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_sod')) +step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') +step_5.add_output('produce') +pipeline_description.add_step(step_5) + +# Step 6: Predictions +step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.construct_predictions.Common')) +step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') +step_6.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_6.add_output('produce') +pipeline_description.add_step(step_6) + +# Final Output +pipeline_description.add_output(name='output predictions', data_reference='steps.6.produce') + +# Output to json +data = pipeline_description.to_json() +with open('example_pipeline.json', 'w') as f: + f.write(data) + print(data) + diff --git a/tods/tests/test_Autocorrelation.py b/tods/tests/test_Autocorrelation.py new file mode 100644 index 0000000..bc82ff3 --- /dev/null +++ b/tods/tests/test_Autocorrelation.py @@ -0,0 +1,134 @@ +import unittest + + +from datetime import datetime + +from d3m import container, utils +from d3m.metadata import base as metadata_base + +from feature_analysis import AutoCorrelation + + +#import utils as test_utils + +import numpy as np +import pandas as pd + +class AutoCorrelationTestCase(unittest.TestCase): + def test_basic(self): + self.maxDiff = None + main = container.DataFrame({'d3mIndex': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + 'timestamp': [1472918400, 1472918700, 1472919000, 1472919300, + 1472919600, 1472919900, 1472920200, 1472920500, 1472920800, 1472921100], + 'value': [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], + 'ground_truth': [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]}, + columns = ['d3mIndex', 'timestamp', 'value', 'ground_truth'], generate_metadata = True) + """ + main.metadata = main.metadata.update_column(0, {'name': 'd3mIndex_'}) + main.metadata = main.metadata.update_column(1, {'name': 'timestamp_'}) + main.metadata = main.metadata.update_column(2, {'name': 'value_'}) + main.metadata = main.metadata.update_column(3, {'name': 'ground_truth_'}) + """ + + #print(main) + + self.assertEqual(utils.to_json_structure(main.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 10, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 4, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'd3mIndex'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'timestamp'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': {'structural_type': 'numpy.float64', 'name': 'value'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 3], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'ground_truth'}, + }]) + + hyperparams_class = AutoCorrelation.AutoCorrelation.metadata.get_hyperparams().defaults() + hyperparams_class = hyperparams_class.replace({'nlags': 2}) + #hyperparams_class = hyperparams_class.replace({'use_semantic_types': True}) + primitive = AutoCorrelation.AutoCorrelation(hyperparams=hyperparams_class) + new_main = primitive.produce(inputs=main).value + + print(new_main) + + new_main_drop = new_main['value_acf'] + new_main_drop = new_main_drop.reset_index(drop = True) + + + expected_result = pd.DataFrame({'acf':[1.000000, 0.700000, 0.412121, 0.148485, -0.078788, -0.257576, -0.375758, -0.421212, -0.381818, -0.245455]}) + + new_main_drop.reset_index() + + self.assertEqual(all(new_main_drop), all(expected_result)) + + + #print(main.metadata.to_internal_simple_structure()) + #print(new_main.metadata.to_internal_simple_structure()) + + self.assertEqual(utils.to_json_structure(main.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 10, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 4, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'd3mIndex'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'timestamp'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': {'structural_type': 'numpy.float64', 'name': 'value'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 3], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'ground_truth'}, + }]) + + params = primitive.get_params() + primitive.set_params(params=params) + + +if __name__ == '__main__': + unittest.main() diff --git a/tods/tests/test_TimeIntervalTransform.py b/tods/tests/test_TimeIntervalTransform.py new file mode 100644 index 0000000..62a48ce --- /dev/null +++ b/tods/tests/test_TimeIntervalTransform.py @@ -0,0 +1,121 @@ +import unittest + + +from datetime import datetime + +from d3m import container, utils +from d3m.metadata import base as metadata_base + +from data_processing import TimeIntervalTransform + + +#import utils as test_utils + +import numpy as np + +class TimeIntervalTransformTestCase(unittest.TestCase): + def test_basic(self): + self.maxDiff = None + main = container.DataFrame({'d3mIndex': [0, 1, 2, 3, 4, 5, 6, 7], + 'timestamp': [1472918400, 1472918700, 1472919000, 1472919300, + 1472919600, 1472919900, 1472920200, 1472920500], + 'value': [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0], + 'ground_truth': [0, 1, 0, 1, 0, 1, 0,1]}, + columns = ['d3mIndex', 'timestamp', 'value', 'ground_truth'], generate_metadata = True) + """ + main.metadata = main.metadata.update_column(0, {'name': 'd3mIndex_'}) + main.metadata = main.metadata.update_column(1, {'name': 'timestamp_'}) + main.metadata = main.metadata.update_column(2, {'name': 'value_'}) + main.metadata = main.metadata.update_column(3, {'name': 'ground_truth_'}) + """ + + #print(main) + + self.assertEqual(utils.to_json_structure(main.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 8, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 4, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'd3mIndex'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'timestamp'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': {'structural_type': 'numpy.float64', 'name': 'value'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 3], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'ground_truth'}, + }]) + + hyperparams_class = TimeIntervalTransform.TimeIntervalTransform.metadata.get_hyperparams() + primitive = TimeIntervalTransform.TimeIntervalTransform(hyperparams=hyperparams_class.defaults()) + new_main = primitive.produce(inputs=main).value + new_rows = len(new_main.index) + self.assertEqual(new_rows, 8) + + #print(main.metadata.to_internal_simple_structure()) + #print(new_main.metadata.to_internal_simple_structure()) + + self.assertEqual(utils.to_json_structure(main.metadata.to_internal_simple_structure()), [{ + 'selector': [], + 'metadata': { + # 'top_level': 'main', + 'schema': metadata_base.CONTAINER_SCHEMA_VERSION, + 'structural_type': 'd3m.container.pandas.DataFrame', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'], + 'dimension': { + 'name': 'rows', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'], + 'length': 8, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__'], + 'metadata': { + 'dimension': { + 'name': 'columns', + 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'], + 'length': 4, + }, + }, + }, { + 'selector': ['__ALL_ELEMENTS__', 0], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'd3mIndex'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 1], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'timestamp'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 2], + 'metadata': {'structural_type': 'numpy.float64', 'name': 'value'}, + }, { + 'selector': ['__ALL_ELEMENTS__', 3], + 'metadata': {'structural_type': 'numpy.int64', 'name': 'ground_truth'}, + }]) + + params = primitive.get_params() + primitive.set_params(params=params) + + +if __name__ == '__main__': + unittest.main() +