From 154cdfea8e2eb438a4236cc0ac864cbada9a6205 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Tue, 10 Nov 2020 00:38:26 -0600 Subject: [PATCH] remove common-primitive dependency, revise primitive_tests Former-commit-id: 160ba3a41db331406a549fa56bae1f9ce026dc81 [formerly 07125c718a9285a8b576fbe8e38a519e2605cda5] [formerly 0f81003685ab707dff628baa9f4b473dcbb7aaa5 [formerly a146528d67cf08c42345bdd49470120508732b54]] [formerly 6162a89a5a16851631ef4ccceb4df8a0d04b01ff [formerly 68cbb6a42e00c1eb23e1941e82e303735de90e7e] [formerly 10cc4b89ce52f1faedb740a8e5da77640ec77849 [formerly e619c2260cf46637a054d722f0459fa0917bbe78]]] [formerly 972f0e1432914bd0939a6d6929aa6fd925b2ad7a [formerly 830ea86dae2fb412fd033f2743914e51c653fb86] [formerly 106d9a0f66a7899fcf928be39906491bcf434dd1 [formerly 3962e5ff748b4ee28ebc18228b3eabd8aa3d61b4]] [formerly bee878ccbfd01caf912c7297bf60649924bb319d [formerly 5adab1c5966aca29f87cba20c1962b243a7ca6f3] [formerly c54a7b24ef34889051807ebeda118d5c444211ac [formerly 0837d0e4462c25f9f98595ee660f8f5061bc331d]]]] [formerly 9fa473030915bccfafce1882df47879e1a76e5cd [formerly d7c7dcd3862fd4b96983a2eac23958506139c154] [formerly f86f25ed8c63375a43a74efc37c01b826ac18ddb [formerly a733830dd8701a9f673e0567a77b45fc88150711]] [formerly e97b53c22a4bfc3aec107c5e4b98967214f057fd [formerly cbf53cf9e6d84c09f10868f4b6a32bd7973c7c52] [formerly f8d34b54a25c9dbdd0b8555dcf8d597ca7174063 [formerly dd5f2e6564402849a28a0f7a2591e8640be1e1db]]] [formerly 03b22869c2832ed2767109236bd60e24539af057 [formerly 58598aa338d335b3b9b452d1f3a1ce27a2378689] [formerly 681ff31fb0566edce835b4df2608e0b66613672d [formerly b7377674fed40cf9416f37a4ff9a0fa6000d57c1]] [formerly 184f35644302785caed1e6fd6ec358794a6d5ed8 [formerly 83d86648d4d7292640dd6931d82e366e6fe2b958] [formerly dace51781aa9a356f5215e06da100a9feb3af641 [formerly 1905e1df459d9383d8e2c637da4165cfb7015f04]]]]] [formerly 4ab9ecea870d9e6ed048fc3a4c67331e8e86724c [formerly b9f81966d516a1e29f9941e2c523b6deb5a402ca] [formerly e84c3e3f44b2e9dac9a1bd8c5f2b110e99a82468 [formerly 8fc402120e852317b93dafe4e8d64b2d411703c7]] [formerly 9764ae63eb6568988be07152457f6a58128a700d [formerly 9966f5263fae5dcca3e2aa035d35537d9dda0f73] [formerly 6583b27757bee26b153704d37f991bea238b4e3b [formerly 169394eb28ae528971ba8efa5b9b1786f6d832f6]]] [formerly 49995d4a88ff269060edca50f81875b83275d9cb [formerly 8e7979866bd05674b5dff98a8ed66c42841acc6d] [formerly 58bea2ac95a2537115d6c589391cb777312cf5cd [formerly 7bcb79d10d5168966dc485ce94cdf4cae5ac4616]] [formerly 3acd90de39338004fe4f853243183abba93bfc30 [formerly 7a92bba6fb042518c2717b0506ce1127a95b66f4] [formerly 252dbf61998179f3ffc6d84a6505f019e56c3f14 [formerly f2edef88147ae669253c6ece6245314a08e61c0f]]]] [formerly a8432b11884152912259e1875bc86790960f9a69 [formerly 7163030193b56d88a613eb1d13866cedd2f0e1b4] [formerly d5dbcc81582b06f7caa0e5a02a9966a04aca43bd [formerly 2c5f43bb9350460644f649a6118ed86d1639a322]] [formerly 525582cafa2181329bbffe0bd7031e2d4696c3c6 [formerly 859c0edef35ab6ddd06cc397c5184b5142b0dd04] [formerly 1141577189f18930d72242ccaf4a88629dae7262 [formerly 96f62655fade6ff11bd7026daf88238d3e1dc159]]] [formerly 714f425f4066c7c6991e5d25427b77dd9db2c99f [formerly f0110252685cecb4b0c38b360bab314e6167182e] [formerly 7b8c741ed1fa11e7846ce34d394878dee83048f6 [formerly 1e1e679c0abf6b9f262c35e335cde4e718f4d6c2]] [formerly 3eb17e347f9ddab025caf79c23b6aa95a5af0b7b [formerly a84f30a47734b9eb2b42309bf09bf5300aead657] [formerly 674df774fc078f6a9059fb113611a4de8fe6b27d [formerly fae7de48dd298648d399d3217ddf25a3116b5065]]]]]] Former-commit-id: a2a07c13835a35a336954fa380cd990092d7956e [formerly 416ec93396dcfb6e45fc1177056aed458f238aa1] [formerly 06fb3d76be784d65b4eaa748a239e22a82a823ef [formerly e76996cbe822540c2a27216f1d42f62cc3b276f8]] [formerly e9ff0e36a3eefe24b0d094e33b17437bfd937d48 [formerly 868c786b3f4240077151555f5866118e8df12b25] [formerly 82b60165d58dd64799f8eab1b9e55c9870c1c206 [formerly 034747c213db980275d99bcc7da5805e86cf510e]]] [formerly babc12a7123c71a4e36e266bc31103bc59209533 [formerly 0a146b28ef80680914e4741cf8d4dfed8c291c54] [formerly 72314677fb739c63f577edea60b2342651062f32 [formerly ff42b8e151b337232baf02ae1ec17ec794f680ac]] [formerly 88b97d74c756e1b4dac38ae2adb054cb98bb9ee2 [formerly 504b50e5478cb5e2741fb781e3bc9fc3a410af9e] [formerly 4b2f6111cecebf6edcb37240b6aa3fb80eb74739 [formerly 802b1a62704bf185026fbefb7f46e735f88848f9]]]] [formerly 0214cb7a3242ba4dce2491aa9f8965e34f75b10d [formerly 5c770977953b7d12c0f34577294b883ae190766a] [formerly b7dd35aac3a2956fd1a210e845981d702f4fbc40 [formerly 11a359eac696c14c84829cd239ba6f523750475f]] [formerly 385645172db92b2b3f4a3bf701b1e693d161eca3 [formerly a793ce79c66f92ac87e3ffe033741cd4429ed0d6] [formerly aa3a910f82a25a9637f570dd322ebd401077736a [formerly d7ba434edcb67dfe944b4cedba65bb56f9361a49]]] [formerly 3c67ce9dcf2394be2387cc5dbb199ebf3c0d33cd [formerly 3291186b5f6ccde9d7ec28d6e7f93bc5f2c23f48] [formerly 8f5992dff364505da7cf18c3ca379caf3e91478f [formerly 9c8a2f8cb60e519fcd1679430a536b3189298cf3]] [formerly cdfbac45be26fc21e54a6b09cf18d3629ef7f6a3 [formerly b4308298d38d8a5ba76b3705970b85c1178fffa7] [formerly 674df774fc078f6a9059fb113611a4de8fe6b27d]]]] Former-commit-id: cee65f78f49406dc34dfcaa58550a97705ad5fb7 [formerly 1332ce886f17a22af4f30b11d7e6119cf99f9a55] [formerly 19fc724aa5d306a37ede95d5daa2564c62026b73 [formerly f03559e414348b7d506d5dcb64b9b8a050f479fa]] [formerly 3f91aa84949831fcb4e8293db4f38d57208c5fea [formerly d8866a78a0ba9231e11d5d3679dccb74da88af30] [formerly 9966d727db4fb2b5d8b172a85440ed6c6bc5a794 [formerly 070aa61aec6f48f9b030c660967b9f2695111d68]]] [formerly a766a963a7f46e8347e52f71bc74dd18e028497d [formerly a244707ec8780d5b338f5285eb3bcf9bf4b7e4fd] [formerly a6beaca84ee9b8300a2c2617e83722f28b864fb3 [formerly 1e24b63e11f72384ad9273e4392beff516336daa]] [formerly 617c1586242f6ac86186817b80d83d8350309131 [formerly 8e22ff2a52663c3ca599223deffbdfa018a4e411] [formerly 83e7b575e7dd9338e53a01e8731a25d75c8491dc [formerly 2516625a718a86a1b3ad70bd6339cb2b7efb9183]]]] Former-commit-id: 7a068f1f72c7515ff90d35453dc5dccd0e40cf71 [formerly 48570c29d118a6b361ccbc2a96ebe5dd6bceffe9] [formerly c86777b7c29bc2662b16a3d5c110acb0ee85e706 [formerly 563d9825b00c32742f4569876c0948e515e87fe1]] [formerly 9a85ff3517a4f4cbbfc3c8ec862ca4212bf62a72 [formerly 7ede1c1e5eeb02ffb9139b082b2c3ac06ac9e184] [formerly f087239b16153e398ea8e5193fd24632d666b1ca [formerly 0aa3080e5fd96cb624af2fc813585f423d5a697e]]] Former-commit-id: d1c39434d1be5a388c55a729d9fd19cfffb4643f [formerly 67be9733bc521fe75593b33d153d2c28622270f5] [formerly ebecc0cbc29aea061a60cd8960e953aeac15724b [formerly 28f45cdda9d5ee7904fc24a8abda74ac8fe2fd6b]] Former-commit-id: 895aa362470ee392ab385050b5b6c997c92d424d [formerly fddc7bee0d52827b009e94d39b21f4db008275b8] Former-commit-id: 5c4c810268965d016fd5b1dd486bceb2843978be --- new_tests/build_ABOD_pipline.py | 70 ------ new_tests/build_AutoEncoder.py | 67 ------ new_tests/build_AutoRegODetect_pipeline.py | 71 ------ new_tests/build_AxiswiseScale_pipline.py | 50 ---- new_tests/build_BKFilter_pipline.py | 44 ---- new_tests/build_CBLOF_pipline.py | 51 ---- new_tests/build_CategoricalToBinary.py | 48 ---- new_tests/build_ColumnFilter_pipeline.py | 49 ---- new_tests/build_ContinuityValidation_pipline.py | 43 ---- new_tests/build_DeepLog_pipeline.py | 49 ---- new_tests/build_DiscreteCosineTransform.py | 50 ---- new_tests/build_DuplicationValidation_pipline.py | 42 ---- new_tests/build_FastFourierTransform.py | 48 ---- new_tests/build_HBOS_pipline.py | 68 ------ new_tests/build_HBOS_score_pipline.py | 71 ------ new_tests/build_HPFilter_pipline.py | 46 ---- new_tests/build_HoltSmoothing_pipline.py | 76 ------ ...uild_HoltWintersExponentialSmoothing_pipline.py | 76 ------ new_tests/build_IsolationForest_pipline.py | 59 ----- new_tests/build_KDiscord_pipeline.py | 71 ------ new_tests/build_KNN_pipline.py | 51 ---- new_tests/build_LODA_pipline.py | 51 ---- new_tests/build_LOF_pipline.py | 51 ---- new_tests/build_LSTMOD_pipline.py | 70 ------ new_tests/build_MatrixProfile_pipeline.py | 49 ---- new_tests/build_MeanAverageTransform_pipline.py | 77 ------ new_tests/build_NonNegativeMatrixFactorization.py | 50 ---- new_tests/build_OCSVM_pipline.py | 51 ---- new_tests/build_PCAODetect_pipeline.py | 71 ------ new_tests/build_PowerTransform_pipline.py | 49 ---- new_tests/build_PyodCOF.py | 51 ---- new_tests/build_QuantileTransform_pipline.py | 49 ---- new_tests/build_RuleBasedFilter_pipline.py | 54 ----- new_tests/build_SOD_pipeline.py | 49 ---- .../build_SimpleExponentialSmoothing_pipline.py | 76 ------ new_tests/build_Standardize_pipline.py | 49 ---- new_tests/build_TRMF_pipline.py | 44 ---- new_tests/build_Telemanom.py | 48 ---- new_tests/build_TimeIntervalTransform_pipeline.py | 86 ------- new_tests/build_TruncatedSVD_pipline.py | 44 ---- new_tests/build_VariationalAutoEncoder.py | 67 ------ new_tests/build_WaveletTransform_pipline.py | 64 ----- .../build_test_detection_algorithm_PyodMoGaal.py | 50 ---- .../build_test_detection_algorithm_PyodSoGaal.py | 50 ---- ...nalysis_spectral_residual_transform_pipeline.py | 61 ----- ...test_feature_analysis_statistical_abs_energy.py | 62 ----- ...ld_test_feature_analysis_statistical_abs_sum.py | 62 ----- ...uild_test_feature_analysis_statistical_gmean.py | 62 ----- ...uild_test_feature_analysis_statistical_hmean.py | 62 ----- ...d_test_feature_analysis_statistical_kurtosis.py | 62 ----- ...ld_test_feature_analysis_statistical_maximum.py | 62 ----- ...build_test_feature_analysis_statistical_mean.py | 62 ----- ...d_test_feature_analysis_statistical_mean_abs.py | 62 ----- ...sis_statistical_mean_abs_temporal_derivative.py | 62 ----- ...nalysis_statistical_mean_temporal_derivative.py | 62 ----- ...ild_test_feature_analysis_statistical_median.py | 62 ----- ...alysis_statistical_median_absolute_deviation.py | 63 ----- ...ld_test_feature_analysis_statistical_minimum.py | 62 ----- ...build_test_feature_analysis_statistical_skew.py | 62 ----- .../build_test_feature_analysis_statistical_std.py | 62 ----- .../build_test_feature_analysis_statistical_var.py | 62 ----- ..._test_feature_analysis_statistical_variation.py | 62 ----- ...ld_test_feature_analysis_statistical_vec_sum.py | 62 ----- ...ture_analysis_statistical_willison_amplitude.py | 62 ----- ...t_feature_analysis_statistical_zero_crossing.py | 62 ----- ..._time_series_seasonality_trend_decomposition.py | 61 ----- primitive_tests/build_ABOD_pipline.py | 6 +- primitive_tests/build_AutoEncoder.py | 6 +- primitive_tests/build_AutoRegODetect_pipeline.py | 4 +- primitive_tests/build_AxiswiseScale_pipline.py | 2 +- primitive_tests/build_BKFilter_pipline.py | 2 +- primitive_tests/build_CBLOF_pipline.py | 2 +- primitive_tests/build_CategoricalToBinary.py | 2 +- primitive_tests/build_ColumnFilter_pipeline.py | 4 +- .../build_ContinuityValidation_pipline.py | 2 +- primitive_tests/build_DeepLog_pipeline.py | 4 +- primitive_tests/build_DiscreteCosineTransform.py | 2 +- .../build_DuplicationValidation_pipline.py | 2 +- primitive_tests/build_FastFourierTransform.py | 2 +- primitive_tests/build_HBOS_pipline.py | 6 +- primitive_tests/build_HBOS_score_pipline.py | 6 +- primitive_tests/build_HPFilter_pipline.py | 4 +- primitive_tests/build_HoltSmoothing_pipline.py | 6 +- ...uild_HoltWintersExponentialSmoothing_pipline.py | 6 +- primitive_tests/build_IsolationForest_pipline.py | 4 +- primitive_tests/build_KDiscord_pipeline.py | 6 +- primitive_tests/build_KNN_pipline.py | 2 +- primitive_tests/build_LODA_pipline.py | 2 +- primitive_tests/build_LOF_pipline.py | 2 +- primitive_tests/build_LSTMOD_pipline.py | 4 +- primitive_tests/build_MatrixProfile_pipeline.py | 2 +- .../build_MeanAverageTransform_pipline.py | 6 +- .../build_NonNegativeMatrixFactorization.py | 2 +- primitive_tests/build_OCSVM_pipline.py | 2 +- primitive_tests/build_PCAODetect_pipeline.py | 6 +- primitive_tests/build_PowerTransform_pipline.py | 2 +- primitive_tests/build_PyodCOF.py | 2 +- primitive_tests/build_QuantileTransform_pipline.py | 2 +- primitive_tests/build_RuleBasedFilter_pipline.py | 4 +- primitive_tests/build_SOD_pipeline.py | 2 +- .../build_SimpleExponentialSmoothing_pipline.py | 6 +- primitive_tests/build_Standardize_pipline.py | 2 +- primitive_tests/build_TRMF_pipline.py | 2 +- primitive_tests/build_Telemanom.py | 2 +- .../build_TimeIntervalTransform_pipeline.py | 8 +- primitive_tests/build_TruncatedSVD_pipline.py | 2 +- primitive_tests/build_VariationalAutoEncoder.py | 6 +- primitive_tests/build_WaveletTransform_pipline.py | 2 +- .../build_test_detection_algorithm_PyodMoGaal.py | 2 +- .../build_test_detection_algorithm_PyodSoGaal.py | 2 +- ...nalysis_spectral_residual_transform_pipeline.py | 4 +- ...test_feature_analysis_statistical_abs_energy.py | 2 +- ...ld_test_feature_analysis_statistical_abs_sum.py | 2 +- ...uild_test_feature_analysis_statistical_gmean.py | 2 +- ...uild_test_feature_analysis_statistical_hmean.py | 4 +- ...d_test_feature_analysis_statistical_kurtosis.py | 4 +- ...ld_test_feature_analysis_statistical_maximum.py | 2 +- ...build_test_feature_analysis_statistical_mean.py | 2 +- ...d_test_feature_analysis_statistical_mean_abs.py | 2 +- ...sis_statistical_mean_abs_temporal_derivative.py | 2 +- ...nalysis_statistical_mean_temporal_derivative.py | 2 +- ...ild_test_feature_analysis_statistical_median.py | 2 +- ...alysis_statistical_median_absolute_deviation.py | 2 +- ...ld_test_feature_analysis_statistical_minimum.py | 2 +- ...build_test_feature_analysis_statistical_skew.py | 2 +- .../build_test_feature_analysis_statistical_std.py | 2 +- .../build_test_feature_analysis_statistical_var.py | 2 +- ..._test_feature_analysis_statistical_variation.py | 2 +- ...ld_test_feature_analysis_statistical_vec_sum.py | 2 +- ...ture_analysis_statistical_willison_amplitude.py | 2 +- ...t_feature_analysis_statistical_zero_crossing.py | 2 +- ..._time_series_seasonality_trend_decomposition.py | 2 +- replace.sh | 9 + test.sh | 2 +- tested_file.txt | 1 - tods/data_processing/CategoricalToBinary.py | 1 - tods/data_processing/ColumnFilter.py | 3 - tods/data_processing/ColumnParser.py | 5 +- tods/data_processing/ConstructPredictions.py | 260 +++++++++++++++++++++ .../ExtractColumnsBySemanticTypes.py | 4 +- tods/data_processing/TimeIntervalTransform.py | 4 +- tods/data_processing/utils.py | 192 +++++++++++++++ tods/feature_analysis/AutoCorrelation.py | 1 - tods/feature_analysis/DiscreteCosineTransform.py | 1 - tods/feature_analysis/FastFourierTransform.py | 1 - tods/feature_analysis/WaveletTransform.py | 3 +- tods/resources/.requirements.txt | 1 - tods/timeseries_processing/SKAxiswiseScaler.py | 1 - 148 files changed, 568 insertions(+), 3986 deletions(-) delete mode 100644 new_tests/build_ABOD_pipline.py delete mode 100644 new_tests/build_AutoEncoder.py delete mode 100644 new_tests/build_AutoRegODetect_pipeline.py delete mode 100644 new_tests/build_AxiswiseScale_pipline.py delete mode 100644 new_tests/build_BKFilter_pipline.py delete mode 100644 new_tests/build_CBLOF_pipline.py delete mode 100644 new_tests/build_CategoricalToBinary.py delete mode 100644 new_tests/build_ColumnFilter_pipeline.py delete mode 100644 new_tests/build_ContinuityValidation_pipline.py delete mode 100644 new_tests/build_DeepLog_pipeline.py delete mode 100644 new_tests/build_DiscreteCosineTransform.py delete mode 100644 new_tests/build_DuplicationValidation_pipline.py delete mode 100644 new_tests/build_FastFourierTransform.py delete mode 100644 new_tests/build_HBOS_pipline.py delete mode 100644 new_tests/build_HBOS_score_pipline.py delete mode 100644 new_tests/build_HPFilter_pipline.py delete mode 100644 new_tests/build_HoltSmoothing_pipline.py delete mode 100644 new_tests/build_HoltWintersExponentialSmoothing_pipline.py delete mode 100644 new_tests/build_IsolationForest_pipline.py delete mode 100644 new_tests/build_KDiscord_pipeline.py delete mode 100644 new_tests/build_KNN_pipline.py delete mode 100644 new_tests/build_LODA_pipline.py delete mode 100644 new_tests/build_LOF_pipline.py delete mode 100644 new_tests/build_LSTMOD_pipline.py delete mode 100644 new_tests/build_MatrixProfile_pipeline.py delete mode 100644 new_tests/build_MeanAverageTransform_pipline.py delete mode 100644 new_tests/build_NonNegativeMatrixFactorization.py delete mode 100644 new_tests/build_OCSVM_pipline.py delete mode 100644 new_tests/build_PCAODetect_pipeline.py delete mode 100644 new_tests/build_PowerTransform_pipline.py delete mode 100644 new_tests/build_PyodCOF.py delete mode 100644 new_tests/build_QuantileTransform_pipline.py delete mode 100644 new_tests/build_RuleBasedFilter_pipline.py delete mode 100644 new_tests/build_SOD_pipeline.py delete mode 100644 new_tests/build_SimpleExponentialSmoothing_pipline.py delete mode 100644 new_tests/build_Standardize_pipline.py delete mode 100644 new_tests/build_TRMF_pipline.py delete mode 100644 new_tests/build_Telemanom.py delete mode 100644 new_tests/build_TimeIntervalTransform_pipeline.py delete mode 100644 new_tests/build_TruncatedSVD_pipline.py delete mode 100644 new_tests/build_VariationalAutoEncoder.py delete mode 100644 new_tests/build_WaveletTransform_pipline.py delete mode 100644 new_tests/build_test_detection_algorithm_PyodMoGaal.py delete mode 100644 new_tests/build_test_detection_algorithm_PyodSoGaal.py delete mode 100644 new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_abs_energy.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_abs_sum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_gmean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_hmean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_kurtosis.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_maximum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_abs.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_median.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_minimum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_skew.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_std.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_var.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_variation.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_vec_sum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_willison_amplitude.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_zero_crossing.py delete mode 100644 new_tests/build_test_time_series_seasonality_trend_decomposition.py create mode 100644 replace.sh delete mode 100644 tested_file.txt create mode 100644 tods/data_processing/ConstructPredictions.py create mode 100644 tods/data_processing/utils.py diff --git a/new_tests/build_ABOD_pipline.py b/new_tests/build_ABOD_pipline.py deleted file mode 100644 index 5faccc2..0000000 --- a/new_tests/build_ABOD_pipline.py +++ /dev/null @@ -1,70 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: ABOD -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_abod')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_5.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2, 4,)) -step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='replace') - -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AutoEncoder.py b/new_tests/build_AutoEncoder.py deleted file mode 100644 index 7482be5..0000000 --- a/new_tests/build_AutoEncoder.py +++ /dev/null @@ -1,67 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: auto encoder -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ae')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AutoRegODetect_pipeline.py b/new_tests/build_AutoRegODetect_pipeline.py deleted file mode 100644 index e6debfa..0000000 --- a/new_tests/build_AutoRegODetect_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.AutoRegODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AxiswiseScale_pipline.py b/new_tests/build_AxiswiseScale_pipline.py deleted file mode 100644 index 3352f48..0000000 --- a/new_tests/build_AxiswiseScale_pipline.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_BKFilter_pipline.py b/new_tests/build_BKFilter_pipline.py deleted file mode 100644 index c2b306f..0000000 --- a/new_tests/build_BKFilter_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: BKFilter -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.bk_filter')) -# step_2.add_hyperparameter(name = 'columns_using_method', argument_type=ArgumentType.VALUE, data = 'name') -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2,3)) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_CBLOF_pipline.py b/new_tests/build_CBLOF_pipline.py deleted file mode 100644 index 2180b6d..0000000 --- a/new_tests/build_CBLOF_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_cblof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_CategoricalToBinary.py b/new_tests/build_CategoricalToBinary.py deleted file mode 100644 index 9f9782e..0000000 --- a/new_tests/build_CategoricalToBinary.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Categorical to Binary -primitive_2 = index.get_primitive('d3m.primitives.tods.data_processing.categorical_to_binary') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(3,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_ColumnFilter_pipeline.py b/new_tests/build_ColumnFilter_pipeline.py deleted file mode 100644 index 3dd3be3..0000000 --- a/new_tests/build_ColumnFilter_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -#Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.auto_correlation') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -primitive_3 = index.get_primitive('d3m.primitives.tods.data_processing.column_filter') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_ContinuityValidation_pipline.py b/new_tests/build_ContinuityValidation_pipline.py deleted file mode 100644 index 3b76d84..0000000 --- a/new_tests/build_ContinuityValidation_pipline.py +++ /dev/null @@ -1,43 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: ContinuityValidation -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.continuity_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'continuity_option', argument_type=ArgumentType.VALUE, data = 'imputation') -step_2.add_hyperparameter(name = 'interval', argument_type=ArgumentType.VALUE, data = 0.3) -# Or: -# step_2.add_hyperparameter(name = 'continuity_option', argument_type=ArgumentType.VALUE, data = 'ablation') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_DeepLog_pipeline.py b/new_tests/build_DeepLog_pipeline.py deleted file mode 100644 index 110c6d3..0000000 --- a/new_tests/build_DeepLog_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.deeplog') - -step_2 = PrimitiveStep(primitive=primitive_2) -#step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_DiscreteCosineTransform.py b/new_tests/build_DiscreteCosineTransform.py deleted file mode 100644 index c052207..0000000 --- a/new_tests/build_DiscreteCosineTransform.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: Discrete Cosine Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.discrete_cosine_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_DuplicationValidation_pipline.py b/new_tests/build_DuplicationValidation_pipline.py deleted file mode 100644 index 57673d2..0000000 --- a/new_tests/build_DuplicationValidation_pipline.py +++ /dev/null @@ -1,42 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: DuplicationValidation -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.duplication_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'keep_option', argument_type=ArgumentType.VALUE, data = 'average') # Or: 'first' -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_FastFourierTransform.py b/new_tests/build_FastFourierTransform.py deleted file mode 100644 index 5c7f083..0000000 --- a/new_tests/build_FastFourierTransform.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Fast Fourier Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.fast_fourier_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_HBOS_pipline.py b/new_tests/build_HBOS_pipline.py deleted file mode 100644 index b281ba0..0000000 --- a/new_tests/build_HBOS_pipline.py +++ /dev/null @@ -1,68 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: HBOS -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_hbos')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -# step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') - -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HBOS_score_pipline.py b/new_tests/build_HBOS_score_pipline.py deleted file mode 100644 index b389a1e..0000000 --- a/new_tests/build_HBOS_score_pipline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: HBOS -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_hbos')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_5.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -# step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') - -step_5.add_output('produce_score') -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') -# pipeline_description.add_output(name='output score', data_reference='steps.5.produce_score') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HPFilter_pipline.py b/new_tests/build_HPFilter_pipline.py deleted file mode 100644 index 355c076..0000000 --- a/new_tests/build_HPFilter_pipline.py +++ /dev/null @@ -1,46 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: HPFilter -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.hp_filter')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') - -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = [2,3,6]) - -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name = 'return_result', argument_type=ArgumentType.VALUE, data = 'append') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HoltSmoothing_pipline.py b/new_tests/build_HoltSmoothing_pipline.py deleted file mode 100644 index 8f8a31e..0000000 --- a/new_tests/build_HoltSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: holt smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.holt_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="exclude_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HoltWintersExponentialSmoothing_pipline.py b/new_tests/build_HoltWintersExponentialSmoothing_pipline.py deleted file mode 100644 index 6ede370..0000000 --- a/new_tests/build_HoltWintersExponentialSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: holt winters exponential smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.holt_winters_exponential_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_IsolationForest_pipline.py b/new_tests/build_IsolationForest_pipline.py deleted file mode 100644 index 80923c9..0000000 --- a/new_tests/build_IsolationForest_pipline.py +++ /dev/null @@ -1,59 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -primitive_3 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_iforest') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -# step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -# step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce_score') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce_score') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_KDiscord_pipeline.py b/new_tests/build_KDiscord_pipeline.py deleted file mode 100644 index 09d6a7c..0000000 --- a/new_tests/build_KDiscord_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.KDiscordODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_KNN_pipline.py b/new_tests/build_KNN_pipline.py deleted file mode 100644 index 8b31557..0000000 --- a/new_tests/build_KNN_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_knn') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LODA_pipline.py b/new_tests/build_LODA_pipline.py deleted file mode 100644 index 05b022d..0000000 --- a/new_tests/build_LODA_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_loda') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LOF_pipline.py b/new_tests/build_LOF_pipline.py deleted file mode 100644 index ec444cf..0000000 --- a/new_tests/build_LOF_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_lof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LSTMOD_pipline.py b/new_tests/build_LSTMOD_pipline.py deleted file mode 100644 index 3575904..0000000 --- a/new_tests/build_LSTMOD_pipline.py +++ /dev/null @@ -1,70 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 2: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 3: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.LSTMODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='diff_group_method', argument_type=ArgumentType.VALUE, data='average') -step_4.add_hyperparameter(name='feature_dim', argument_type=ArgumentType.VALUE, data=5) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_MatrixProfile_pipeline.py b/new_tests/build_MatrixProfile_pipeline.py deleted file mode 100644 index 458823e..0000000 --- a/new_tests/build_MatrixProfile_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.matrix_profile') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=3) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_MeanAverageTransform_pipline.py b/new_tests/build_MeanAverageTransform_pipline.py deleted file mode 100644 index 43bf392..0000000 --- a/new_tests/build_MeanAverageTransform_pipline.py +++ /dev/null @@ -1,77 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: mean average transform -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.moving_average_transform')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_NonNegativeMatrixFactorization.py b/new_tests/build_NonNegativeMatrixFactorization.py deleted file mode 100644 index 787013c..0000000 --- a/new_tests/build_NonNegativeMatrixFactorization.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Non Negative Matrix Factorization -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.non_negative_matrix_factorization') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_hyperparameter(name='rank', argument_type=ArgumentType.VALUE, data=5) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_OCSVM_pipline.py b/new_tests/build_OCSVM_pipline.py deleted file mode 100644 index d8cd8c9..0000000 --- a/new_tests/build_OCSVM_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ocsvm') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PCAODetect_pipeline.py b/new_tests/build_PCAODetect_pipeline.py deleted file mode 100644 index 327cacd..0000000 --- a/new_tests/build_PCAODetect_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.PCAODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PowerTransform_pipline.py b/new_tests/build_PowerTransform_pipline.py deleted file mode 100644 index b855dc7..0000000 --- a/new_tests/build_PowerTransform_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.power_transformer') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PyodCOF.py b/new_tests/build_PyodCOF.py deleted file mode 100644 index fcd0d2b..0000000 --- a/new_tests/build_PyodCOF.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_cof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_QuantileTransform_pipline.py b/new_tests/build_QuantileTransform_pipline.py deleted file mode 100644 index f6c4868..0000000 --- a/new_tests/build_QuantileTransform_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.quantile_transformer') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_RuleBasedFilter_pipline.py b/new_tests/build_RuleBasedFilter_pipline.py deleted file mode 100644 index 87a74b9..0000000 --- a/new_tests/build_RuleBasedFilter_pipline.py +++ /dev/null @@ -1,54 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - - -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.reinforcement.rule_filter')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') - -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2, 4,)) -step_3.add_hyperparameter(name='rule', argument_type=ArgumentType.VALUE, data='#4# % 2 == 0 and #2# <= 0.3') - -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -pipeline_description.add_step(step_3) - - - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_SOD_pipeline.py b/new_tests/build_SOD_pipeline.py deleted file mode 100644 index e4ed1b3..0000000 --- a/new_tests/build_SOD_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_sod') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_SimpleExponentialSmoothing_pipline.py b/new_tests/build_SimpleExponentialSmoothing_pipline.py deleted file mode 100644 index b33db22..0000000 --- a/new_tests/build_SimpleExponentialSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: simple exponential smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.simple_exponential_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (1,)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_Standardize_pipline.py b/new_tests/build_Standardize_pipline.py deleted file mode 100644 index 8300d7c..0000000 --- a/new_tests/build_Standardize_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_TRMF_pipline.py b/new_tests/build_TRMF_pipline.py deleted file mode 100644 index 7d7c407..0000000 --- a/new_tests/build_TRMF_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: TRMF -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.trmf')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') - -step_2.add_hyperparameter(name = 'lags', argument_type=ArgumentType.VALUE, data = [1,2,10,100]) -# step_2.add_hyperparameter(name = 'K', argument_type=ArgumentType.VALUE, data = 3) -# step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2, 3, 4, 5, 6)) - -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_Telemanom.py b/new_tests/build_Telemanom.py deleted file mode 100644 index 06a192c..0000000 --- a/new_tests/build_Telemanom.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Fast Fourier Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.telemanom') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_TimeIntervalTransform_pipeline.py b/new_tests/build_TimeIntervalTransform_pipeline.py deleted file mode 100644 index be7990f..0000000 --- a/new_tests/build_TimeIntervalTransform_pipeline.py +++ /dev/null @@ -1,86 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: dataframe transformation -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKPowerTransformer') -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKStandardization') -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKQuantileTransformer') - -#Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -primitive_2 = index.get_primitive('d3m.primitives.tods.data_processing.time_interval_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name="time_interval", argument_type=ArgumentType.VALUE, data = '5T') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) -# -# # Step 2: column_parser -# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -# step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -# step_2.add_output('produce') -# pipeline_description.add_step(step_2) -# -# -# # Step 3: extract_columns_by_semantic_types(attributes) -# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -# step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -# step_3.add_output('produce') -# step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, -# data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -# pipeline_description.add_step(step_3) -# -# # Step 4: extract_columns_by_semantic_types(targets) -# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -# step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -# step_4.add_output('produce') -# step_4.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, -# data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -# pipeline_description.add_step(step_4) -# -# attributes = 'steps.3.produce' -# targets = 'steps.4.produce' -# -# # Step 5: imputer -# step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_cleaning.imputer.SKlearn')) -# step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -# step_5.add_output('produce') -# pipeline_description.add_step(step_5) -# -# # Step 6: random_forest -# step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.regression.random_forest.SKlearn')) -# step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -# step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -# step_6.add_output('produce') -# pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.1.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_TruncatedSVD_pipline.py b/new_tests/build_TruncatedSVD_pipline.py deleted file mode 100644 index 290f181..0000000 --- a/new_tests/build_TruncatedSVD_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: TruncatedSVD -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.truncated_svd')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'n_components', argument_type=ArgumentType.VALUE, data = 3) -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2, 3, 4, 5, 6)) -step_2.add_hyperparameter(name = 'return_result', argument_type=ArgumentType.VALUE, data = 'append') -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_VariationalAutoEncoder.py b/new_tests/build_VariationalAutoEncoder.py deleted file mode 100644 index e585a0a..0000000 --- a/new_tests/build_VariationalAutoEncoder.py +++ /dev/null @@ -1,67 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: variatinal auto encoder -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_vae')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_WaveletTransform_pipline.py b/new_tests/build_WaveletTransform_pipline.py deleted file mode 100644 index ee6c766..0000000 --- a/new_tests/build_WaveletTransform_pipline.py +++ /dev/null @@ -1,64 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test WaveletTransform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.wavelet_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='wavelet', argument_type=ArgumentType.VALUE, data='db8') -step_2.add_hyperparameter(name='level', argument_type=ArgumentType.VALUE, data=2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 2: test inverse WaveletTransform -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.wavelet_transform') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='wavelet', argument_type=ArgumentType.VALUE, data='db8') -step_3.add_hyperparameter(name='level', argument_type=ArgumentType.VALUE, data=2) -step_3.add_hyperparameter(name='inverse', argument_type=ArgumentType.VALUE, data=1) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_detection_algorithm_PyodMoGaal.py b/new_tests/build_test_detection_algorithm_PyodMoGaal.py deleted file mode 100644 index 713a2cd..0000000 --- a/new_tests/build_test_detection_algorithm_PyodMoGaal.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_mogaal') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_test_detection_algorithm_PyodSoGaal.py b/new_tests/build_test_detection_algorithm_PyodSoGaal.py deleted file mode 100644 index 4caa752..0000000 --- a/new_tests/build_test_detection_algorithm_PyodSoGaal.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_sogaal') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py b/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py deleted file mode 100644 index 5fbd61e..0000000 --- a/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py +++ /dev/null @@ -1,61 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.spectral_residual_transform') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='avg_filter_dimension', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_abs_energy.py b/new_tests/build_test_feature_analysis_statistical_abs_energy.py deleted file mode 100644 index cb28366..0000000 --- a/new_tests/build_test_feature_analysis_statistical_abs_energy.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_abs_energy') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_abs_sum.py b/new_tests/build_test_feature_analysis_statistical_abs_sum.py deleted file mode 100644 index 91b3d42..0000000 --- a/new_tests/build_test_feature_analysis_statistical_abs_sum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_abs_sum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_gmean.py b/new_tests/build_test_feature_analysis_statistical_gmean.py deleted file mode 100644 index 5d54b3c..0000000 --- a/new_tests/build_test_feature_analysis_statistical_gmean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_g_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_hmean.py b/new_tests/build_test_feature_analysis_statistical_hmean.py deleted file mode 100644 index 01f9dbb..0000000 --- a/new_tests/build_test_feature_analysis_statistical_hmean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_h_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_kurtosis.py b/new_tests/build_test_feature_analysis_statistical_kurtosis.py deleted file mode 100644 index 3276152..0000000 --- a/new_tests/build_test_feature_analysis_statistical_kurtosis.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_kurtosis') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_maximum.py b/new_tests/build_test_feature_analysis_statistical_maximum.py deleted file mode 100644 index 900a5c1..0000000 --- a/new_tests/build_test_feature_analysis_statistical_maximum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_maximum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean.py b/new_tests/build_test_feature_analysis_statistical_mean.py deleted file mode 100644 index 29c7bb0..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_abs.py b/new_tests/build_test_feature_analysis_statistical_mean_abs.py deleted file mode 100644 index 6be3c45..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_abs.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_abs') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py b/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py deleted file mode 100644 index 15c12aa..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_abs_temporal_derivative') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py b/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py deleted file mode 100644 index d63dddb..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_temporal_derivative') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_median.py b/new_tests/build_test_feature_analysis_statistical_median.py deleted file mode 100644 index cefe002..0000000 --- a/new_tests/build_test_feature_analysis_statistical_median.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_median') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py b/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py deleted file mode 100644 index 499a877..0000000 --- a/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py +++ /dev/null @@ -1,63 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_median_abs_deviation') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_minimum.py b/new_tests/build_test_feature_analysis_statistical_minimum.py deleted file mode 100644 index 01c918d..0000000 --- a/new_tests/build_test_feature_analysis_statistical_minimum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_minimum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_skew.py b/new_tests/build_test_feature_analysis_statistical_skew.py deleted file mode 100644 index 7ca113c..0000000 --- a/new_tests/build_test_feature_analysis_statistical_skew.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_skew') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_std.py b/new_tests/build_test_feature_analysis_statistical_std.py deleted file mode 100644 index 66d3180..0000000 --- a/new_tests/build_test_feature_analysis_statistical_std.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_std') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_var.py b/new_tests/build_test_feature_analysis_statistical_var.py deleted file mode 100644 index bd13e96..0000000 --- a/new_tests/build_test_feature_analysis_statistical_var.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_var') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_variation.py b/new_tests/build_test_feature_analysis_statistical_variation.py deleted file mode 100644 index 5292e03..0000000 --- a/new_tests/build_test_feature_analysis_statistical_variation.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_variation') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_vec_sum.py b/new_tests/build_test_feature_analysis_statistical_vec_sum.py deleted file mode 100644 index fa8f99b..0000000 --- a/new_tests/build_test_feature_analysis_statistical_vec_sum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_vec_sum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py b/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py deleted file mode 100644 index f750dad..0000000 --- a/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_willison_amplitude') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_zero_crossing.py b/new_tests/build_test_feature_analysis_statistical_zero_crossing.py deleted file mode 100644 index 1c4efa1..0000000 --- a/new_tests/build_test_feature_analysis_statistical_zero_crossing.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_zero_crossing') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(9,10)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_time_series_seasonality_trend_decomposition.py b/new_tests/build_test_time_series_seasonality_trend_decomposition.py deleted file mode 100644 index ab172bf..0000000 --- a/new_tests/build_test_time_series_seasonality_trend_decomposition.py +++ /dev/null @@ -1,61 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.decomposition.time_series_seasonality_trend_decomposition') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='period', argument_type=ArgumentType.VALUE, data=5) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/primitive_tests/build_ABOD_pipline.py b/primitive_tests/build_ABOD_pipline.py index b48328b..5faccc2 100644 --- a/primitive_tests/build_ABOD_pipline.py +++ b/primitive_tests/build_ABOD_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_AutoEncoder.py b/primitive_tests/build_AutoEncoder.py index 7e05df1..7482be5 100644 --- a/primitive_tests/build_AutoEncoder.py +++ b/primitive_tests/build_AutoEncoder.py @@ -16,13 +16,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -30,7 +30,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_AutoRegODetect_pipeline.py b/primitive_tests/build_AutoRegODetect_pipeline.py index 1034b97..e6debfa 100644 --- a/primitive_tests/build_AutoRegODetect_pipeline.py +++ b/primitive_tests/build_AutoRegODetect_pipeline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_AxiswiseScale_pipline.py b/primitive_tests/build_AxiswiseScale_pipline.py index 7c51ab2..3352f48 100644 --- a/primitive_tests/build_AxiswiseScale_pipline.py +++ b/primitive_tests/build_AxiswiseScale_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_BKFilter_pipline.py b/primitive_tests/build_BKFilter_pipline.py index e68fb68..c2b306f 100644 --- a/primitive_tests/build_BKFilter_pipline.py +++ b/primitive_tests/build_BKFilter_pipline.py @@ -15,7 +15,7 @@ pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_CBLOF_pipline.py b/primitive_tests/build_CBLOF_pipline.py index 302727d..2180b6d 100644 --- a/primitive_tests/build_CBLOF_pipline.py +++ b/primitive_tests/build_CBLOF_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_CategoricalToBinary.py b/primitive_tests/build_CategoricalToBinary.py index a29399c..9f9782e 100644 --- a/primitive_tests/build_CategoricalToBinary.py +++ b/primitive_tests/build_CategoricalToBinary.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_ColumnFilter_pipeline.py b/primitive_tests/build_ColumnFilter_pipeline.py index 9afa446..7c1aa55 100644 --- a/primitive_tests/build_ColumnFilter_pipeline.py +++ b/primitive_tests/build_ColumnFilter_pipeline.py @@ -10,14 +10,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) #Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_ContinuityValidation_pipline.py b/primitive_tests/build_ContinuityValidation_pipline.py index c42310b..3b76d84 100644 --- a/primitive_tests/build_ContinuityValidation_pipline.py +++ b/primitive_tests/build_ContinuityValidation_pipline.py @@ -13,7 +13,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_DeepLog_pipeline.py b/primitive_tests/build_DeepLog_pipeline.py index 32c69d0..0ab8fa3 100644 --- a/primitive_tests/build_DeepLog_pipeline.py +++ b/primitive_tests/build_DeepLog_pipeline.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_DiscreteCosineTransform.py b/primitive_tests/build_DiscreteCosineTransform.py index c3cc52f..c052207 100644 --- a/primitive_tests/build_DiscreteCosineTransform.py +++ b/primitive_tests/build_DiscreteCosineTransform.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_DuplicationValidation_pipline.py b/primitive_tests/build_DuplicationValidation_pipline.py index 6471ac9..57673d2 100644 --- a/primitive_tests/build_DuplicationValidation_pipline.py +++ b/primitive_tests/build_DuplicationValidation_pipline.py @@ -15,7 +15,7 @@ pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_FastFourierTransform.py b/primitive_tests/build_FastFourierTransform.py index 10a8914..5c7f083 100644 --- a/primitive_tests/build_FastFourierTransform.py +++ b/primitive_tests/build_FastFourierTransform.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_HBOS_pipline.py b/primitive_tests/build_HBOS_pipline.py index 2ebe958..b281ba0 100644 --- a/primitive_tests/build_HBOS_pipline.py +++ b/primitive_tests/build_HBOS_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HBOS_score_pipline.py b/primitive_tests/build_HBOS_score_pipline.py index 6e3cd4a..b389a1e 100644 --- a/primitive_tests/build_HBOS_score_pipline.py +++ b/primitive_tests/build_HBOS_score_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HPFilter_pipline.py b/primitive_tests/build_HPFilter_pipline.py index 80754bf..fbf6941 100644 --- a/primitive_tests/build_HPFilter_pipline.py +++ b/primitive_tests/build_HPFilter_pipline.py @@ -8,14 +8,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_HoltSmoothing_pipline.py b/primitive_tests/build_HoltSmoothing_pipline.py index 374be8b..8f8a31e 100644 --- a/primitive_tests/build_HoltSmoothing_pipline.py +++ b/primitive_tests/build_HoltSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py b/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py index b275408..6ede370 100644 --- a/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py +++ b/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_IsolationForest_pipline.py b/primitive_tests/build_IsolationForest_pipline.py index 965343c..80923c9 100644 --- a/primitive_tests/build_IsolationForest_pipline.py +++ b/primitive_tests/build_IsolationForest_pipline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_KDiscord_pipeline.py b/primitive_tests/build_KDiscord_pipeline.py index e66bcb1..fc12db9 100644 --- a/primitive_tests/build_KDiscord_pipeline.py +++ b/primitive_tests/build_KDiscord_pipeline.py @@ -12,21 +12,21 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_KNN_pipline.py b/primitive_tests/build_KNN_pipline.py index d188c76..8b31557 100644 --- a/primitive_tests/build_KNN_pipline.py +++ b/primitive_tests/build_KNN_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LODA_pipline.py b/primitive_tests/build_LODA_pipline.py index daac9f6..05b022d 100644 --- a/primitive_tests/build_LODA_pipline.py +++ b/primitive_tests/build_LODA_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LOF_pipline.py b/primitive_tests/build_LOF_pipline.py index 0c82d13..ec444cf 100644 --- a/primitive_tests/build_LOF_pipline.py +++ b/primitive_tests/build_LOF_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LSTMOD_pipline.py b/primitive_tests/build_LSTMOD_pipline.py index 0e9a54b..3575904 100644 --- a/primitive_tests/build_LSTMOD_pipline.py +++ b/primitive_tests/build_LSTMOD_pipline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_MatrixProfile_pipeline.py b/primitive_tests/build_MatrixProfile_pipeline.py index 41ceeaa..458823e 100644 --- a/primitive_tests/build_MatrixProfile_pipeline.py +++ b/primitive_tests/build_MatrixProfile_pipeline.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_MeanAverageTransform_pipline.py b/primitive_tests/build_MeanAverageTransform_pipline.py index 223c631..43bf392 100644 --- a/primitive_tests/build_MeanAverageTransform_pipline.py +++ b/primitive_tests/build_MeanAverageTransform_pipline.py @@ -17,14 +17,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -32,7 +32,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_NonNegativeMatrixFactorization.py b/primitive_tests/build_NonNegativeMatrixFactorization.py index bdfcd2f..787013c 100644 --- a/primitive_tests/build_NonNegativeMatrixFactorization.py +++ b/primitive_tests/build_NonNegativeMatrixFactorization.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_OCSVM_pipline.py b/primitive_tests/build_OCSVM_pipline.py index 640fb9b..d8cd8c9 100644 --- a/primitive_tests/build_OCSVM_pipline.py +++ b/primitive_tests/build_OCSVM_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_PCAODetect_pipeline.py b/primitive_tests/build_PCAODetect_pipeline.py index 1e93027..1c7da0e 100644 --- a/primitive_tests/build_PCAODetect_pipeline.py +++ b/primitive_tests/build_PCAODetect_pipeline.py @@ -12,21 +12,21 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_PowerTransform_pipline.py b/primitive_tests/build_PowerTransform_pipline.py index 94ecfbd..b855dc7 100644 --- a/primitive_tests/build_PowerTransform_pipline.py +++ b/primitive_tests/build_PowerTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_PyodCOF.py b/primitive_tests/build_PyodCOF.py index 7128e6d..fcd0d2b 100644 --- a/primitive_tests/build_PyodCOF.py +++ b/primitive_tests/build_PyodCOF.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_QuantileTransform_pipline.py b/primitive_tests/build_QuantileTransform_pipline.py index 28c4911..f6c4868 100644 --- a/primitive_tests/build_QuantileTransform_pipline.py +++ b/primitive_tests/build_QuantileTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_RuleBasedFilter_pipline.py b/primitive_tests/build_RuleBasedFilter_pipline.py index 013af40..87a74b9 100644 --- a/primitive_tests/build_RuleBasedFilter_pipline.py +++ b/primitive_tests/build_RuleBasedFilter_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_SOD_pipeline.py b/primitive_tests/build_SOD_pipeline.py index 8858cee..e4ed1b3 100644 --- a/primitive_tests/build_SOD_pipeline.py +++ b/primitive_tests/build_SOD_pipeline.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_SimpleExponentialSmoothing_pipline.py b/primitive_tests/build_SimpleExponentialSmoothing_pipline.py index 6f35df3..b33db22 100644 --- a/primitive_tests/build_SimpleExponentialSmoothing_pipline.py +++ b/primitive_tests/build_SimpleExponentialSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_Standardize_pipline.py b/primitive_tests/build_Standardize_pipline.py index 50844ea..8300d7c 100644 --- a/primitive_tests/build_Standardize_pipline.py +++ b/primitive_tests/build_Standardize_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_TRMF_pipline.py b/primitive_tests/build_TRMF_pipline.py index b058cbc..7d7c407 100644 --- a/primitive_tests/build_TRMF_pipline.py +++ b/primitive_tests/build_TRMF_pipline.py @@ -14,7 +14,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_Telemanom.py b/primitive_tests/build_Telemanom.py index afb4bb3..06a192c 100644 --- a/primitive_tests/build_Telemanom.py +++ b/primitive_tests/build_Telemanom.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_TimeIntervalTransform_pipeline.py b/primitive_tests/build_TimeIntervalTransform_pipeline.py index 289171c..be7990f 100644 --- a/primitive_tests/build_TimeIntervalTransform_pipeline.py +++ b/primitive_tests/build_TimeIntervalTransform_pipeline.py @@ -22,7 +22,7 @@ pipeline_description.add_step(step_0) # primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKQuantileTransformer') #Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) @@ -35,14 +35,14 @@ step_2.add_output('produce') pipeline_description.add_step(step_2) # # # Step 2: column_parser -# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) # step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') # step_2.add_output('produce') # pipeline_description.add_step(step_2) # # # # Step 3: extract_columns_by_semantic_types(attributes) -# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) # step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') # step_3.add_output('produce') # step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -50,7 +50,7 @@ pipeline_description.add_step(step_2) # pipeline_description.add_step(step_3) # # # Step 4: extract_columns_by_semantic_types(targets) -# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) # step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') # step_4.add_output('produce') # step_4.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_TruncatedSVD_pipline.py b/primitive_tests/build_TruncatedSVD_pipline.py index af14404..290f181 100644 --- a/primitive_tests/build_TruncatedSVD_pipline.py +++ b/primitive_tests/build_TruncatedSVD_pipline.py @@ -14,7 +14,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_VariationalAutoEncoder.py b/primitive_tests/build_VariationalAutoEncoder.py index 01031be..e585a0a 100644 --- a/primitive_tests/build_VariationalAutoEncoder.py +++ b/primitive_tests/build_VariationalAutoEncoder.py @@ -16,13 +16,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -30,7 +30,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_WaveletTransform_pipline.py b/primitive_tests/build_WaveletTransform_pipline.py index de3aa90..ee6c766 100644 --- a/primitive_tests/build_WaveletTransform_pipline.py +++ b/primitive_tests/build_WaveletTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py b/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py index a488900..713a2cd 100644 --- a/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py +++ b/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py b/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py index f14c5a0..4caa752 100644 --- a/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py +++ b/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py b/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py index d481651..6278460 100644 --- a/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py +++ b/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py b/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py index fc616c4..cb28366 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py +++ b/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py b/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py index 2a5f499..91b3d42 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_gmean.py b/primitive_tests/build_test_feature_analysis_statistical_gmean.py index 05230ff..5d54b3c 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_gmean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_gmean.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_hmean.py b/primitive_tests/build_test_feature_analysis_statistical_hmean.py index 047a6c4..19fa5b2 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_hmean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_hmean.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py b/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py index ef5ea92..b6b01a7 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py +++ b/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_maximum.py b/primitive_tests/build_test_feature_analysis_statistical_maximum.py index e3bb764..900a5c1 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_maximum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_maximum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean.py b/primitive_tests/build_test_feature_analysis_statistical_mean.py index 5112fec..29c7bb0 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py b/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py index 4c94ec6..6be3c45 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py b/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py index a1bdcdd..15c12aa 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py b/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py index e785efb..d63dddb 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_median.py b/primitive_tests/build_test_feature_analysis_statistical_median.py index b11b108..cefe002 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_median.py +++ b/primitive_tests/build_test_feature_analysis_statistical_median.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py b/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py index e219ad7..499a877 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py +++ b/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_minimum.py b/primitive_tests/build_test_feature_analysis_statistical_minimum.py index be8a148..01c918d 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_minimum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_minimum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_skew.py b/primitive_tests/build_test_feature_analysis_statistical_skew.py index b506f03..7ca113c 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_skew.py +++ b/primitive_tests/build_test_feature_analysis_statistical_skew.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_std.py b/primitive_tests/build_test_feature_analysis_statistical_std.py index f7a61d7..66d3180 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_std.py +++ b/primitive_tests/build_test_feature_analysis_statistical_std.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_var.py b/primitive_tests/build_test_feature_analysis_statistical_var.py index 386c932..bd13e96 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_var.py +++ b/primitive_tests/build_test_feature_analysis_statistical_var.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_variation.py b/primitive_tests/build_test_feature_analysis_statistical_variation.py index bc1f680..5292e03 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_variation.py +++ b/primitive_tests/build_test_feature_analysis_statistical_variation.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py b/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py index 85f06d2..fa8f99b 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py b/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py index 6c578fc..f750dad 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py +++ b/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py b/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py index 7a33424..1c4efa1 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py +++ b/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py b/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py index 99168bf..ab172bf 100644 --- a/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py +++ b/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/replace.sh b/replace.sh new file mode 100644 index 0000000..2b0b986 --- /dev/null +++ b/replace.sh @@ -0,0 +1,9 @@ +# !/bin/bash + +files=$(ls primitive_tests) +for f in $files +do + f_path="./primitive_tests/"$f + save_path="./new_tests/"$f + cat $f_path | sed 's/d3m.primitives.data_transformation.dataset_to_dataframe.Common/d3m.primitives.tods.data_processing.dataset_to_dataframe/g'| sed 's/d3m.primitives.data_transformation.column_parser.Common/d3m.primitives.tods.data_processing.column_parser/g' | sed 's/d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common/d3m.primitives.tods.data_processing.extract_columns_by_semantic_types/g' | sed 's/d3m.primitives.data_transformation.construct_predictions.Common/d3m.primitives.tods.data_processing.construct_predictions/g' > $save_path +done diff --git a/test.sh b/test.sh index 3481545..9316de2 100644 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -test_scripts=$(ls new_tests) +test_scripts=$(ls primitiver_tests) #test_scripts=$(ls primitive_tests | grep -v -f tested_file.txt) for file in $test_scripts diff --git a/tested_file.txt b/tested_file.txt deleted file mode 100644 index 6672770..0000000 --- a/tested_file.txt +++ /dev/null @@ -1 +0,0 @@ -build_ABOD_pipline.py diff --git a/tods/data_processing/CategoricalToBinary.py b/tods/data_processing/CategoricalToBinary.py index f6dd7d5..1dad198 100644 --- a/tods/data_processing/CategoricalToBinary.py +++ b/tods/data_processing/CategoricalToBinary.py @@ -11,7 +11,6 @@ from d3m.primitive_interfaces import base, transformer from d3m.primitive_interfaces.base import CallResult, DockerContainer -import common_primitives import logging import math diff --git a/tods/data_processing/ColumnFilter.py b/tods/data_processing/ColumnFilter.py index 9c14a75..f16d24f 100644 --- a/tods/data_processing/ColumnFilter.py +++ b/tods/data_processing/ColumnFilter.py @@ -14,11 +14,8 @@ import logging, uuid from scipy import sparse from numpy import ndarray from collections import OrderedDict -from common_primitives import dataframe_utils, utils -from d3m import utils from d3m import container -from d3m.base import utils as base_utils from d3m.exceptions import PrimitiveNotFittedError from d3m.container import DataFrame as d3m_dataframe from d3m.container.numpy import ndarray as d3m_ndarray diff --git a/tods/data_processing/ColumnParser.py b/tods/data_processing/ColumnParser.py index c552ab5..37088a4 100644 --- a/tods/data_processing/ColumnParser.py +++ b/tods/data_processing/ColumnParser.py @@ -9,8 +9,7 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives -from common_primitives import utils +from tods.data_processing import utils __all__ = ('ColumnParserPrimitive',) @@ -98,7 +97,7 @@ class ColumnParserPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs 'name': "Parses strings into their types", 'python_path': 'd3m.primitives.tods.data_processing.column_parser', 'source': { - 'name': common_primitives.__author__, + 'name': "DataLab@Texas A&M University", 'contact': 'mailto:mitar.commonprimitives@tnode.com', 'uris': [ 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/column_parser.py', diff --git a/tods/data_processing/ConstructPredictions.py b/tods/data_processing/ConstructPredictions.py new file mode 100644 index 0000000..37bc57f --- /dev/null +++ b/tods/data_processing/ConstructPredictions.py @@ -0,0 +1,260 @@ +import os +import typing + +from d3m import container, utils as d3m_utils +from d3m.metadata import base as metadata_base, hyperparams +from d3m.primitive_interfaces import base, transformer +from d3m.contrib.primitives import compute_scores + + +__all__ = ('ConstructPredictionsPrimitive',) + +Inputs = container.DataFrame +Outputs = container.DataFrame + + +class Hyperparams(hyperparams.Hyperparams): + use_columns = hyperparams.Set( + elements=hyperparams.Hyperparameter[int](-1), + default=(), + semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], + description="A set of column indices to force primitive to operate on. If metadata reconstruction happens, this is used for reference columns." + " If any specified column is not a primary index or a predicted target, it is skipped.", + ) + exclude_columns = hyperparams.Set( + elements=hyperparams.Hyperparameter[int](-1), + default=(), + semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], + description="A set of column indices to not operate on. If metadata reconstruction happens, this is used for reference columns. Applicable only if \"use_columns\" is not provided.", + ) + + +class ConstructPredictionsPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): + """ + A primitive which takes as input a DataFrame and outputs a DataFrame in Lincoln Labs predictions + format: first column is a d3mIndex column (and other primary index columns, e.g., for object detection + problem), and then predicted targets, each in its column, followed by optional confidence column(s). + + It supports both input columns annotated with semantic types (``https://metadata.datadrivendiscovery.org/types/PrimaryKey``, + ``https://metadata.datadrivendiscovery.org/types/PrimaryMultiKey``, ``https://metadata.datadrivendiscovery.org/types/PredictedTarget``, + ``https://metadata.datadrivendiscovery.org/types/Confidence``), or trying to reconstruct metadata. + This is why the primitive takes also additional input of a reference DataFrame which should + have metadata to help reconstruct missing metadata. If metadata is missing, the primitive + assumes that all ``inputs`` columns are predicted targets, without confidence column(s). + """ + + metadata = metadata_base.PrimitiveMetadata( + { + 'id': '8d38b340-f83f-4877-baaa-162f8e551736', + 'version': '0.3.0', + 'name': "Construct pipeline predictions output", + 'python_path': 'd3m.primitives.tods.data_processing.construct_predictions', + 'source': { + 'name': "DataLab@Texas A&M University", + 'contact': 'mailto:mitar.commonprimitives@tnode.com', + 'uris': [ + 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/construct_predictions.py', + 'https://gitlab.com/datadrivendiscovery/common-primitives.git', + ], + }, + 'installation': [{ + 'type': metadata_base.PrimitiveInstallationType.PIP, + 'package_uri': 'git+https://gitlab.com/datadrivendiscovery/common-primitives.git@{git_commit}#egg=common_primitives'.format( + git_commit=d3m_utils.current_git_commit(os.path.dirname(__file__)), + ), + }], + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.DATA_CONVERSION, + ], + 'primitive_family': metadata_base.PrimitiveFamily.DATA_TRANSFORMATION, + }, + ) + + def produce(self, *, inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.CallResult[Outputs]: # type: ignore + index_columns = inputs.metadata.get_index_columns() + target_columns = inputs.metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/PredictedTarget',)) + + # Target columns cannot be also index columns. This should not really happen, + # but it could happen with buggy primitives. + target_columns = [target_column for target_column in target_columns if target_column not in index_columns] + + if index_columns and target_columns: + outputs = self._produce_using_semantic_types(inputs, index_columns, target_columns) + else: + outputs = self._produce_reconstruct(inputs, reference, index_columns, target_columns) + + outputs = compute_scores.ComputeScoresPrimitive._encode_columns(outputs) + + # Generally we do not care about column names in DataFrame itself (but use names of columns from metadata), + # but in this case setting column names makes it easier to assure that "to_csv" call produces correct output. + # See: https://gitlab.com/datadrivendiscovery/d3m/issues/147 + column_names = [] + for column_index in range(len(outputs.columns)): + column_names.append(outputs.metadata.query_column(column_index).get('name', outputs.columns[column_index])) + outputs.columns = column_names + + return base.CallResult(outputs) + + def _filter_index_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int]) -> typing.Sequence[int]: + if self.hyperparams['use_columns']: + index_columns = [index_column_index for index_column_index in index_columns if index_column_index in self.hyperparams['use_columns']] + if not index_columns: + raise ValueError("No index columns listed in \"use_columns\" hyper-parameter, but index columns are required.") + + else: + index_columns = [index_column_index for index_column_index in index_columns if index_column_index not in self.hyperparams['exclude_columns']] + if not index_columns: + raise ValueError("All index columns listed in \"exclude_columns\" hyper-parameter, but index columns are required.") + + names = [] + for index_column in index_columns: + index_metadata = inputs_metadata.query_column(index_column) + # We do not care about empty strings for names either. + if index_metadata.get('name', None): + names.append(index_metadata['name']) + + if 'd3mIndex' not in names: + raise ValueError("\"d3mIndex\" index column is missing.") + + names_set = set(names) + if len(names) != len(names_set): + duplicate_names = names + for name in names_set: + # Removes just the first occurrence. + duplicate_names.remove(name) + + self.logger.warning("Duplicate names for index columns: %(duplicate_names)s", { + 'duplicate_names': list(set(duplicate_names)), + }) + + return index_columns + + def _get_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> typing.List[int]: + assert index_columns + assert target_columns + + index_columns = self._filter_index_columns(inputs_metadata, index_columns) + + if self.hyperparams['use_columns']: + target_columns = [target_column_index for target_column_index in target_columns if target_column_index in self.hyperparams['use_columns']] + if not target_columns: + raise ValueError("No target columns listed in \"use_columns\" hyper-parameter, but target columns are required.") + + else: + target_columns = [target_column_index for target_column_index in target_columns if target_column_index not in self.hyperparams['exclude_columns']] + if not target_columns: + raise ValueError("All target columns listed in \"exclude_columns\" hyper-parameter, but target columns are required.") + + assert index_columns + assert target_columns + + return list(index_columns) + list(target_columns) + + def _get_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata) -> typing.List[int]: + confidence_columns = inputs_metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/Confidence',)) + + if self.hyperparams['use_columns']: + confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index in self.hyperparams['use_columns']] + else: + confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index not in self.hyperparams['exclude_columns']] + + return confidence_columns + + def _produce_using_semantic_types(self, inputs: Inputs, index_columns: typing.Sequence[int], + target_columns: typing.Sequence[int]) -> Outputs: + confidence_columns = self._get_confidence_columns(inputs.metadata) + + output_columns = self._get_columns(inputs.metadata, index_columns, target_columns) + confidence_columns + + # "get_index_columns" makes sure that "d3mIndex" is always listed first. + # And "select_columns" selects columns in order listed, which then + # always puts "d3mIndex" first. + outputs = inputs.select_columns(output_columns) + + if confidence_columns: + outputs.metadata = self._update_confidence_columns(outputs.metadata, confidence_columns) + + return outputs + + def _update_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata, confidence_columns: typing.Sequence[int]) -> metadata_base.DataMetadata: + output_columns_length = inputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + outputs_metadata = inputs_metadata + + # All confidence columns have to be named "confidence". + for column_index in range(output_columns_length - len(confidence_columns), output_columns_length): + outputs_metadata = outputs_metadata.update((metadata_base.ALL_ELEMENTS, column_index), { + 'name': 'confidence', + }) + + return outputs_metadata + + def _produce_reconstruct(self, inputs: Inputs, reference: Inputs, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> Outputs: + if not index_columns: + reference_index_columns = reference.metadata.get_index_columns() + + if not reference_index_columns: + raise ValueError("Cannot find an index column in reference data, but index column is required.") + + filtered_index_columns = self._filter_index_columns(reference.metadata, reference_index_columns) + index = reference.select_columns(filtered_index_columns) + else: + filtered_index_columns = self._filter_index_columns(inputs.metadata, index_columns) + index = inputs.select_columns(filtered_index_columns) + + if not target_columns: + if index_columns: + raise ValueError("No target columns in input data, but index column(s) present.") + + # We assume all inputs are targets. + targets = inputs + + # We make sure at least basic metadata is generated correctly, so we regenerate metadata. + targets.metadata = targets.metadata.generate(targets) + + # We set target column names from the reference. We set semantic types. + targets.metadata = self._update_targets_metadata(targets.metadata, self._get_target_names(reference.metadata)) + + else: + targets = inputs.select_columns(target_columns) + + return index.append_columns(targets) + + def multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore + return self._multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) + + def fit_multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore + return self._fit_multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) + + def _get_target_names(self, metadata: metadata_base.DataMetadata) -> typing.List[typing.Union[str, None]]: + target_names = [] + + for column_index in metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/TrueTarget',)): + column_metadata = metadata.query((metadata_base.ALL_ELEMENTS, column_index)) + + target_names.append(column_metadata.get('name', None)) + + return target_names + + def _update_targets_metadata(self, metadata: metadata_base.DataMetadata, target_names: typing.Sequence[typing.Union[str, None]]) -> metadata_base.DataMetadata: + targets_length = metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + if targets_length != len(target_names): + raise ValueError("Not an expected number of target columns to apply names for. Expected {target_names}, provided {targets_length}.".format( + target_names=len(target_names), + targets_length=targets_length, + )) + + for column_index, target_name in enumerate(target_names): + metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/Target') + metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/PredictedTarget') + + # We do not have it, let's skip it and hope for the best. + if target_name is None: + continue + + metadata = metadata.update_column(column_index, { + 'name': target_name, + }) + + return metadata diff --git a/tods/data_processing/ExtractColumnsBySemanticTypes.py b/tods/data_processing/ExtractColumnsBySemanticTypes.py index 283cb18..76d9676 100644 --- a/tods/data_processing/ExtractColumnsBySemanticTypes.py +++ b/tods/data_processing/ExtractColumnsBySemanticTypes.py @@ -6,8 +6,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives - __all__ = ('ExtractColumnsBySemanticTypesPrimitive',) Inputs = container.DataFrame @@ -74,7 +72,7 @@ class ExtractColumnsBySemanticTypesPrimitive(transformer.TransformerPrimitiveBas 'name': "Extracts columns by semantic type", 'python_path': 'd3m.primitives.tods.data_processing.extract_columns_by_semantic_types', 'source': { - 'name': common_primitives.__author__, + 'name': "DataLab@Texas A&M University", 'contact': 'mailto:mitar.commonprimitives@tnode.com', 'uris': [ 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/extract_columns_semantic_types.py', diff --git a/tods/data_processing/TimeIntervalTransform.py b/tods/data_processing/TimeIntervalTransform.py index 3231215..c202bf7 100644 --- a/tods/data_processing/TimeIntervalTransform.py +++ b/tods/data_processing/TimeIntervalTransform.py @@ -6,12 +6,10 @@ import collections import numpy as np import pandas as pd -import common_primitives -from common_primitives import dataframe_utils, utils from datetime import datetime, timezone from d3m.primitive_interfaces import base, transformer -from d3m import container, exceptions, utils as d3m_utils +from d3m import container from d3m.metadata import base as metadata_base, hyperparams diff --git a/tods/data_processing/utils.py b/tods/data_processing/utils.py new file mode 100644 index 0000000..d45dbf0 --- /dev/null +++ b/tods/data_processing/utils.py @@ -0,0 +1,192 @@ +import datetime +import logging +import typing + +import dateutil.parser +import numpy # type: ignore + +from d3m import container, deprecate +from d3m.base import utils as base_utils +from d3m.metadata import base as metadata_base + +logger = logging.getLogger(__name__) + +DEFAULT_DATETIME = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + + +@deprecate.function(message="it should not be used anymore") +def copy_elements_metadata(source_metadata: metadata_base.Metadata, target_metadata: metadata_base.DataMetadata, from_selector: metadata_base.Selector, + to_selector: metadata_base.Selector = (), *, ignore_all_elements: bool = False, check: bool = True, source: typing.Any = None) -> metadata_base.DataMetadata: + return source_metadata._copy_elements_metadata(target_metadata, list(from_selector), list(to_selector), [], ignore_all_elements) + + +@deprecate.function(message="use Metadata.copy_to method instead") +def copy_metadata(source_metadata: metadata_base.Metadata, target_metadata: metadata_base.DataMetadata, from_selector: metadata_base.Selector, + to_selector: metadata_base.Selector = (), *, ignore_all_elements: bool = False, check: bool = True, source: typing.Any = None) -> metadata_base.DataMetadata: + return source_metadata.copy_to(target_metadata, from_selector, to_selector, ignore_all_elements=ignore_all_elements) + + +@deprecate.function(message="use DataFrame.select_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def select_columns(inputs: container.DataFrame, columns: typing.Sequence[metadata_base.SimpleSelectorSegment], *, + source: typing.Any = None) -> container.DataFrame: + return inputs.select_columns(columns) + + +@deprecate.function(message="use DataMetadata.select_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def select_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns: typing.Sequence[metadata_base.SimpleSelectorSegment], *, + source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.select_columns(columns) + + +@deprecate.function(message="use DataMetadata.list_columns_with_semantic_types method instead") +def list_columns_with_semantic_types(metadata: metadata_base.DataMetadata, semantic_types: typing.Sequence[str], *, + at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.list_columns_with_semantic_types(semantic_types, at=at) + + +@deprecate.function(message="use DataMetadata.list_columns_with_structural_types method instead") +def list_columns_with_structural_types(metadata: metadata_base.DataMetadata, structural_types: typing.Union[typing.Callable, typing.Sequence[typing.Union[str, type]]], *, + at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.list_columns_with_structural_types(structural_types, at=at) + + +@deprecate.function(message="use DataFrame.remove_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def remove_columns(inputs: container.DataFrame, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> container.DataFrame: + return inputs.remove_columns(column_indices) + + +@deprecate.function(message="use DataMetadata.remove_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def remove_columns_metadata(inputs_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.remove_columns(column_indices) + + +@deprecate.function(message="use DataFrame.append_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def append_columns(left: container.DataFrame, right: container.DataFrame, *, use_right_metadata: bool = False, source: typing.Any = None) -> container.DataFrame: + return left.append_columns(right, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataMetadata.append_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def append_columns_metadata(left_metadata: metadata_base.DataMetadata, right_metadata: metadata_base.DataMetadata, use_right_metadata: bool = False, source: typing.Any = None) -> metadata_base.DataMetadata: + return left_metadata.append_columns(right_metadata, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataFrame.insert_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def insert_columns(inputs: container.DataFrame, columns: container.DataFrame, at_column_index: int, *, source: typing.Any = None) -> container.DataFrame: + return inputs.insert_columns(columns, at_column_index) + + +@deprecate.function(message="use DataMetadata.insert_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def insert_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns_metadata: metadata_base.DataMetadata, at_column_index: int, *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.insert_columns(columns_metadata, at_column_index) + + +@deprecate.function(message="use DataFrame.replace_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def replace_columns(inputs: container.DataFrame, columns: container.DataFrame, column_indices: typing.Sequence[int], *, copy: bool = True, source: typing.Any = None) -> container.DataFrame: + return inputs.replace_columns(columns, column_indices, copy=copy) + + +@deprecate.function(message="use DataMetadata.replace_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def replace_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.replace_columns(columns_metadata, column_indices) + + +@deprecate.function(message="use DataMetadata.get_index_columns method instead") +def get_index_columns(metadata: metadata_base.DataMetadata, *, at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.get_index_columns(at=at) + + +@deprecate.function(message="use DataFrame.horizontal_concat method instead") +@deprecate.arguments('source', message="argument ignored") +def horizontal_concat(left: container.DataFrame, right: container.DataFrame, *, use_index: bool = True, + remove_second_index: bool = True, use_right_metadata: bool = False, source: typing.Any = None) -> container.DataFrame: + return left.horizontal_concat(right, use_index=use_index, remove_second_index=remove_second_index, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataMetadata.horizontal_concat method instead") +@deprecate.arguments('source', message="argument ignored") +def horizontal_concat_metadata(left_metadata: metadata_base.DataMetadata, right_metadata: metadata_base.DataMetadata, *, use_index: bool = True, + remove_second_index: bool = True, use_right_metadata: bool = False, source: typing.Any = None) -> metadata_base.DataMetadata: + return left_metadata.horizontal_concat(right_metadata, use_index=use_index, remove_second_index=remove_second_index, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use d3m.base.utils.get_columns_to_use function instead") +def get_columns_to_use(metadata: metadata_base.DataMetadata, use_columns: typing.Sequence[int], exclude_columns: typing.Sequence[int], + can_use_column: typing.Callable) -> typing.Tuple[typing.List[int], typing.List[int]]: + return base_utils.get_columns_to_use(metadata, use_columns, exclude_columns, can_use_column) + + +@deprecate.function(message="use d3m.base.utils.combine_columns function instead") +@deprecate.arguments('source', message="argument ignored") +def combine_columns(return_result: str, add_index_columns: bool, inputs: container.DataFrame, column_indices: typing.Sequence[int], + columns_list: typing.Sequence[container.DataFrame], *, source: typing.Any = None) -> container.DataFrame: + return base_utils.combine_columns(inputs, column_indices, columns_list, return_result=return_result, add_index_columns=add_index_columns) + + +@deprecate.function(message="use d3m.base.utils.combine_columns_metadata function instead") +@deprecate.arguments('source', message="argument ignored") +def combine_columns_metadata(return_result: str, add_index_columns: bool, inputs_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], + columns_metadata_list: typing.Sequence[metadata_base.DataMetadata], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return base_utils.combine_columns_metadata(inputs_metadata, column_indices, columns_metadata_list, return_result=return_result, add_index_columns=add_index_columns) + + +@deprecate.function(message="use DataMetadata.set_table_metadata method instead") +@deprecate.arguments('source', message="argument ignored") +def set_table_metadata(inputs_metadata: metadata_base.DataMetadata, *, at: metadata_base.Selector = (), source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.set_table_metadata(at=at) + + +@deprecate.function(message="use DataMetadata.get_column_index_from_column_name method instead") +def get_column_index_from_column_name(inputs_metadata: metadata_base.DataMetadata, column_name: str, *, at: metadata_base.Selector = ()) -> int: + return inputs_metadata.get_column_index_from_column_name(column_name, at=at) + + +@deprecate.function(message="use Dataset.get_relations_graph method instead") +def build_relation_graph(dataset: container.Dataset) -> typing.Dict[str, typing.List[typing.Tuple[str, bool, int, int, typing.Dict]]]: + return dataset.get_relations_graph() + + +@deprecate.function(message="use d3m.base.utils.get_tabular_resource function instead") +def get_tabular_resource(dataset: container.Dataset, resource_id: typing.Optional[str], *, + pick_entry_point: bool = True, pick_one: bool = True, has_hyperparameter: bool = True) -> typing.Tuple[str, container.DataFrame]: + return base_utils.get_tabular_resource(dataset, resource_id, pick_entry_point=pick_entry_point, pick_one=pick_one, has_hyperparameter=has_hyperparameter) + + +@deprecate.function(message="use d3m.base.utils.get_tabular_resource_metadata function instead") +def get_tabular_resource_metadata(dataset_metadata: metadata_base.DataMetadata, resource_id: typing.Optional[metadata_base.SelectorSegment], *, + pick_entry_point: bool = True, pick_one: bool = True) -> metadata_base.SelectorSegment: + return base_utils.get_tabular_resource_metadata(dataset_metadata, resource_id, pick_entry_point=pick_entry_point, pick_one=pick_one) + + +@deprecate.function(message="use Dataset.select_rows method instead") +@deprecate.arguments('source', message="argument ignored") +def cut_dataset(dataset: container.Dataset, row_indices_to_keep: typing.Mapping[str, typing.Sequence[int]], *, + source: typing.Any = None) -> container.Dataset: + return dataset.select_rows(row_indices_to_keep) + + +def parse_datetime(value: str, *, fuzzy: bool = True) -> typing.Optional[datetime.datetime]: + try: + return dateutil.parser.parse(value, default=DEFAULT_DATETIME, fuzzy=fuzzy) + except (ValueError, OverflowError, TypeError): + return None + + +def parse_datetime_to_float(value: str, *, fuzzy: bool = True) -> float: + try: + parsed = parse_datetime(value, fuzzy=fuzzy) + if parsed is None: + return numpy.nan + else: + return parsed.timestamp() + except (ValueError, OverflowError, TypeError): + return numpy.nan diff --git a/tods/feature_analysis/AutoCorrelation.py b/tods/feature_analysis/AutoCorrelation.py index 021996d..2fe2bae 100644 --- a/tods/feature_analysis/AutoCorrelation.py +++ b/tods/feature_analysis/AutoCorrelation.py @@ -14,7 +14,6 @@ import logging, uuid from scipy import sparse from numpy import ndarray from collections import OrderedDict -from common_primitives import dataframe_utils, utils from d3m import utils from d3m import container diff --git a/tods/feature_analysis/DiscreteCosineTransform.py b/tods/feature_analysis/DiscreteCosineTransform.py index 77387ec..88b141e 100644 --- a/tods/feature_analysis/DiscreteCosineTransform.py +++ b/tods/feature_analysis/DiscreteCosineTransform.py @@ -8,7 +8,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives import logging import math from scipy.fft import dct diff --git a/tods/feature_analysis/FastFourierTransform.py b/tods/feature_analysis/FastFourierTransform.py index 2ce4d69..a1bfdea 100644 --- a/tods/feature_analysis/FastFourierTransform.py +++ b/tods/feature_analysis/FastFourierTransform.py @@ -8,7 +8,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives import logging from cmath import polar from scipy.fft import fft diff --git a/tods/feature_analysis/WaveletTransform.py b/tods/feature_analysis/WaveletTransform.py index 61758b3..00ea7d6 100644 --- a/tods/feature_analysis/WaveletTransform.py +++ b/tods/feature_analysis/WaveletTransform.py @@ -3,8 +3,8 @@ import typing from numpy import ndarray import numpy as np -from d3m import container, utils as d3m_utils from d3m.base import utils as base_utils +from d3m import container, exceptions from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer from typing import Union @@ -13,7 +13,6 @@ import pywt import pandas import math -import common_primitives import numpy from typing import Optional, List from collections import OrderedDict diff --git a/tods/resources/.requirements.txt b/tods/resources/.requirements.txt index b773c1d..8e2dd00 100644 --- a/tods/resources/.requirements.txt +++ b/tods/resources/.requirements.txt @@ -1,4 +1,3 @@ numpy==1.18.2 -e git+https://github.com/tods-doc/d3m@70aeefed6b7307941581357c4b7858bb3f88e1da#egg=d3m -e git+https://github.com/tods-doc/axolotl@af54e6970476a081bf0cd65990c9f56a1200d8a2#egg=axolotl --e git+https://gitlab.com/datadrivendiscovery/common-primitives.git@046b20d2f6d4543dcbe18f0a1d4bcbb1f61cf518#egg=common_primitives diff --git a/tods/timeseries_processing/SKAxiswiseScaler.py b/tods/timeseries_processing/SKAxiswiseScaler.py index a18e1e3..d34de79 100644 --- a/tods/timeseries_processing/SKAxiswiseScaler.py +++ b/tods/timeseries_processing/SKAxiswiseScaler.py @@ -9,7 +9,6 @@ from d3m.primitive_interfaces import base, transformer from sklearn.preprocessing import scale -import common_primitives import numpy from typing import Optional, List from collections import OrderedDict