add ensemble methods to pipeline

Former-commit-id: 0aac8ed31d [formerly 54095e5d5f] [formerly 5f0e21e200 [formerly 9702ef05e7]] [formerly 08c89fc468 [formerly 62d490094c] [formerly ea2a04f2d7 [formerly e9198d0f19]]] [formerly b5781dc6ef [formerly 2ced0513ec] [formerly 116f617063 [formerly 7b649b3cd2]] [formerly 0fa9117014 [formerly 9c32f76780] [formerly 23897889c5 [formerly bfce13eeab]]]] [formerly e88ebc108f [formerly 8e0da546a6] [formerly 5c399fc9e2 [formerly 79e41ea33e]] [formerly aa1974a08b [formerly 823f9638a9] [formerly c617b22e67 [formerly ecb36514a0]]] [formerly 0010947336 [formerly d119c90230] [formerly 52bb0ce978 [formerly 4026124a79]] [formerly 35f7ba2d2e [formerly 713166e029] [formerly a624cee295 [formerly 4929310a7f]]]]] [formerly 74a828bb11 [formerly e7f0c2e563] [formerly 490afa890c [formerly 9206b73b90]] [formerly 8ca01b1230 [formerly 15e949f788] [formerly baf4e0150a [formerly 04053bae35]]] [formerly 8a9428f968 [formerly db62e7894d] [formerly 715b0046f3 [formerly 59b2aed435]] [formerly 635186ce7b [formerly cc0cf93bb6] [formerly 9ca2a4e006 [formerly c3d79d1cd0]]]] [formerly f46290d582 [formerly 4716012839] [formerly 4ba202f0e0 [formerly 3856e5eb2a]] [formerly deb75358b7 [formerly 0d99e8fa0b] [formerly 3ce5b6f093 [formerly a61c4bb9de]]] [formerly c7039765bb [formerly dadcd9915b] [formerly 08a186707e [formerly d1bf05d7e9]] [formerly fcb2590086 [formerly 5dfb75e6ed] [formerly 9308f51cbc [formerly 427e33c78c]]]]]]
Former-commit-id: e8ef4326af [formerly 380372c0fb] [formerly 93871265c8 [formerly 3aff99cde7]] [formerly 6d892c0d24 [formerly 1d8eb6d2cc] [formerly 16f625b708 [formerly dc1f8c55bf]]] [formerly ca9d83ffa3 [formerly c09a6d106b] [formerly 2e3317f357 [formerly 977d8b3029]] [formerly 370448792d [formerly f92f5e8e9b] [formerly f74e823892 [formerly 83cc7e2d6c]]]] [formerly 7abc24c7a7 [formerly 27d525a99e] [formerly bc6b671a0e [formerly a9ea335e33]] [formerly b00a5949ff [formerly 152bf4c388] [formerly 135e8bf531 [formerly 90cbe27565]]] [formerly 8ba2b02175 [formerly e5117f2da4] [formerly 32ad600dd4 [formerly f92251497e]] [formerly e224e6dbc3 [formerly c0a682e018] [formerly 9308f51cbc]]]]
Former-commit-id: c15d9ba7fe [formerly b607deb294] [formerly 434d0e154b [formerly cb55cd1547]] [formerly b51280175c [formerly c1a7fc5a7b] [formerly 64397fd023 [formerly cc7b7afc75]]] [formerly 4549e6b126 [formerly 9a5f6e86b2] [formerly 468b0a7b59 [formerly f58d568719]] [formerly cd891c7725 [formerly f5c3a4daf6] [formerly a5236160c8 [formerly 31e5383ba1]]]]
Former-commit-id: dd6fde427a [formerly e6b828625d] [formerly 20376456d0 [formerly 2f19f94b6c]] [formerly 9b23f2e069 [formerly 872ac94733] [formerly f8325526d6 [formerly 18439b2511]]]
Former-commit-id: 3364cf5232 [formerly 084c4f4f61] [formerly 1c7610b381 [formerly 0b10d322cc]]
Former-commit-id: 8ba5d34080 [formerly 69b539f6c0]
Former-commit-id: 1d3c2449e2

examples/build_Ensemble.py

@@ -0,0 +1,72 @@
+from d3m import index
+from d3m.metadata.base import ArgumentType
+from d3m.metadata.pipeline import Pipeline, PrimitiveStep
+
+# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest
+#                                             extract_columns_by_semantic_types(targets)    ->            ^
+
+# Creating pipeline
+pipeline_description = Pipeline()
+pipeline_description.add_input(name='inputs')
+
+# Step 0: dataset_to_dataframe
+step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common'))
+step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0')
+step_0.add_output('produce')
+pipeline_description.add_step(step_0)
+
+# Step 1: column_parser
+step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common'))
+step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce')
+step_1.add_output('produce')
+pipeline_description.add_step(step_1)
+
+# Step 2: extract_columns_by_semantic_types(attributes)
+step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common'))
+step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce')
+step_2.add_output('produce')
+step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE,
+                                  data=['https://metadata.datadrivendiscovery.org/types/Attribute'])
+pipeline_description.add_step(step_2)
+
+# Step 3: extract_columns_by_semantic_types(targets)
+step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common'))
+step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce')
+step_3.add_output('produce')
+step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE,
+                                  data=['https://metadata.datadrivendiscovery.org/types/TrueTarget'])
+pipeline_description.add_step(step_3)
+
+attributes = 'steps.2.produce'
+targets = 'steps.3.produce'
+
+# Step 4: auto encoder
+step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ae'))
+step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes)
+step_4.add_output('produce_score')
+step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=[0,1,2])
+step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True)
+step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append')
+pipeline_description.add_step(step_4)
+
+# Step 5: ensemble
+step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.Ensemble'))
+step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce_score')
+step_5.add_output('produce')
+pipeline_description.add_step(step_5)
+
+
+# Final Output
+pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce')
+
+# Output to YAML
+#yaml = pipeline_description.to_yaml()
+#with open('pipeline.yml', 'w') as f:
+#    f.write(yaml)
+#prin(yaml)
+
+# Output to json
+data = pipeline_description.to_json()
+with open('example_pipeline.json', 'w') as f:
+    f.write(data)
+    print(data)