OpenI
/
tods
Not watched
1
0
Fork 0
Code
Releases 0 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c9cfcea65f
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c9cfcea65f'
${ noResults }
tods/examples/axolotl_interface/example_pipelines/script/build_system_pipeline.py

94 lines
5.0 kB
Raw Blame History 

  1. import argparse

  2. 

  3. from d3m import index

  4. from d3m.metadata.base import ArgumentType

  5. from d3m.metadata.pipeline import Pipeline, PrimitiveStep

  6. 

  7. 

  8. # Creating pipeline

  9. pipeline_description = Pipeline()

  10. pipeline_description.add_input(name='inputs')

  11. 

  12. #Step 0: Denormalise

  13. step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.common.denormalize'))

  14. step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0')

  15. step_0.add_output('produce')

  16. pipeline_description.add_step(step_0)

  17. 

  18. #Step 1: Convert the dataset to a DataFrame

  19. step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe'))

  20. step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce')

  21. step_1.add_output('produce')

  22. pipeline_description.add_step(step_1)

  23. 

  24. #Step 2: Read the csvs corresponding to the paths in the Dataframe in the form of arrays

  25. step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.common.csv_reader'))

  26. step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce')

  27. step_2.add_output('produce')

  28. step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = [0,1])

  29. step_2.add_hyperparameter(name = 'return_result', argument_type=ArgumentType.VALUE, data = 'replace')

  30. pipeline_description.add_step(step_2)

  31. 

  32. #Step 3: Column Parser

  33. step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser'))

  34. step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce')

  35. step_3.add_output('produce')

  36. step_3.add_hyperparameter(name='parse_semantic_types', argument_type=ArgumentType.VALUE,

  37.                                   data=['http://schema.org/Boolean','http://schema.org/Integer', 'http://schema.org/Float', 'https://metadata.datadrivendiscovery.org/types/FloatVector',])

  38. pipeline_description.add_step(step_3)

  39. 

  40. # Step 4: extract_columns_by_semantic_types(attributes)

  41. step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types'))

  42. step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce')

  43. step_4.add_output('produce')

  44. step_4.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE,

  45. 							  data=['https://metadata.datadrivendiscovery.org/types/Attribute'])

  46. pipeline_description.add_step(step_4)

  47. 

  48. # Step 5: extract_columns_by_semantic_types(targets)

  49. step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types'))

  50. step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce')

  51. step_5.add_output('produce')

  52. step_5.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE,

  53. 							data=['https://metadata.datadrivendiscovery.org/types/TrueTarget'])

  54. pipeline_description.add_step(step_5)

  55. 

  56. attributes = 'steps.4.produce'

  57. targets = 'steps.5.produce'

  58. 

  59. # Step 6: processing

  60. step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_maximum'))

  61. step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes)

  62. step_6.add_output('produce')

  63. pipeline_description.add_step(step_6)

  64. 

  65. # Step 7: algorithm

  66. #step_7 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ae'))

  67. step_7 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ocsvm'))

  68. step_7.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.6.produce')

  69. step_7.add_output('produce_score')

  70. pipeline_description.add_step(step_7)

  71. 

  72. # Step 8: Predictions

  73. #step_8 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.construct_predictions'))

  74. step_8 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.system_wise_detection'))

  75. step_8.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.7.produce_score')

  76. #step_8.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce')

  77. step_8.add_output('produce')

  78. pipeline_description.add_step(step_8)

  79. 

  80. step_9 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.construct_predictions'))

  81. step_9.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.8.produce')

  82. step_9.add_argument(name='reference', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce')

  83. step_9.add_output('produce')

  84. pipeline_description.add_step(step_9)

  85. 

  86. # Final Output

  87. pipeline_description.add_output(name='output predictions', data_reference='steps.9.produce')

  88. 

  89. # Output to json

  90. data = pipeline_description.to_json()

  91. with open('system_pipeline.json', 'w') as f:

  92.     f.write(data)

  93.     print(data)