You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

davinci_model.cc 160 kB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
4 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
5 years ago
5 years ago
4 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960
  1. /**
  2. * Copyright 2019-2020 Huawei Technologies Co., Ltd
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. #include "graph/load/new_model_manager/davinci_model.h"
  17. #include <cce/dnn.h>
  18. #include <dlfcn.h>
  19. #include <graph/utils/node_utils.h>
  20. #include <pthread.h>
  21. #include <sched.h>
  22. #include <sys/prctl.h>
  23. #include <algorithm>
  24. #include <map>
  25. #include <utility>
  26. #include "common/debug/log.h"
  27. #include "common/formats/formats.h"
  28. #include "common/formats/utils/formats_trans_utils.h"
  29. #include "common/math/math_util.h"
  30. #include "common/op/ge_op_utils.h"
  31. #include "common/profiling/profiling_manager.h"
  32. #include "common/properties_manager.h"
  33. #include "common/scope_guard.h"
  34. #include "common/thread_pool.h"
  35. #include "framework/common/debug/ge_log.h"
  36. #include "graph/common/ge_call_wrapper.h"
  37. #include "graph/compute_graph.h"
  38. #include "graph/debug/ge_attr_define.h"
  39. #include "graph/ge_context.h"
  40. #include "graph/graph.h"
  41. #include "graph/load/new_model_manager/cpu_queue_schedule.h"
  42. #include "graph/load/new_model_manager/model_manager.h"
  43. #include "graph/load/new_model_manager/tbe_handle_store.h"
  44. #include "graph/manager/graph_mem_allocator.h"
  45. #include "graph/manager/graph_var_manager.h"
  46. #include "graph/manager/trans_var_data_utils.h"
  47. #include "graph/manager/util/debug.h"
  48. #include "graph/model_serialize.h"
  49. #include "graph/node.h"
  50. #include "graph/utils/graph_utils.h"
  51. #include "graph/utils/type_utils.h"
  52. #include "init/gelib.h"
  53. #include "mmpa/mmpa_api.h"
  54. #include "omm/csa_interact.h"
  55. #include "runtime/base.h"
  56. #include "runtime/dev.h"
  57. #include "runtime/event.h"
  58. #include "runtime/mem.h"
  59. #include "runtime/rt_model.h"
  60. #include "runtime/stream.h"
  61. #include "securec.h"
  62. // create std::thread, catch exceptions using try/catch
  63. #define CREATE_STD_THREAD(thread_id, func, args) \
  64. do { \
  65. try { \
  66. thread_id = std::thread(func, args); \
  67. } catch (const std::system_error &e) { \
  68. GELOGE(FAILED, "Caught system_error with code:%d, meaning:%s", e.code().value(), e.what()); \
  69. GELOGE(FAILED, "Thread creat FAIL, Please check the left resource!"); \
  70. return FAILED; \
  71. } \
  72. } while (0)
  73. namespace ge {
  74. namespace {
  75. const uint32_t kDataIndex = 0;
  76. const uint32_t kOutputNum = 1;
  77. const uint32_t kTrueBranchStreamNum = 1;
  78. const uint32_t kThreadNum = 16;
  79. const uint32_t kAddrLen = sizeof(void *);
  80. const int kDecimal = 10;
  81. const int kBytes = 8;
  82. const uint32_t kDataMemAlignSizeCompare = 64;
  83. const uint32_t kDumpL1FusionOpMByteSize = 2 * 1024 * 1024;
  84. const uint32_t kDumpFlagOfL1Fusion = 0;
  85. const char *const kDefaultBatchLable = "Batch_default";
  86. inline bool IsDataOp(const std::string &node_type) {
  87. return node_type == DATA_TYPE || node_type == AIPP_DATA_TYPE || node_type == ANN_DATA_TYPE;
  88. }
  89. inline bool IsNoTaskAndDumpNeeded(const OpDescPtr &op_desc) {
  90. bool save_dump_info = false;
  91. (void)ge::AttrUtils::GetBool(op_desc, ATTR_NO_TASK_AND_DUMP_NEEDED, save_dump_info);
  92. return save_dump_info;
  93. }
  94. } // namespace
  95. std::mutex DavinciModel::tvm_bin_mutex_;
  96. DavinciModel::DavinciModel(int32_t priority, const std::shared_ptr<ModelListener> &listener)
  97. : weights_mem_base_(nullptr),
  98. var_mem_base_(nullptr),
  99. mem_base_(nullptr),
  100. is_inner_mem_base_(false),
  101. is_inner_weight_base_(false),
  102. is_inner_p2p_mem_base_(false),
  103. data_inputer_(nullptr),
  104. load_begin_time_(0),
  105. load_end_time_(0),
  106. time_info_(),
  107. dataInputTid(0),
  108. is_model_has_inited_(false),
  109. model_id_(0),
  110. runtime_model_id_(0),
  111. version_(0),
  112. ge_model_(nullptr),
  113. thread_id_(),
  114. listener_(listener),
  115. run_flg_(false),
  116. priority_(priority),
  117. rt_model_handle_(nullptr),
  118. rt_model_stream_(nullptr),
  119. is_inner_model_stream_(false),
  120. is_async_mode_(false),
  121. last_execute_mode_(INITIALIZATION),
  122. session_id_(0),
  123. device_id_(0),
  124. maxDumpOpNum_(0), data_dumper_(runtime_param_),
  125. iterator_count_(0),
  126. is_l1_fusion_enable_(false),
  127. is_first_execute_(true) {
  128. op_list_.clear();
  129. }
  130. DavinciModel::~DavinciModel() {
  131. try {
  132. Status ret = data_dumper_.UnloadDumpInfo();
  133. if (ret != SUCCESS) {
  134. GELOGW("UnloadDumpInfo failed, ret: %u.", ret);
  135. }
  136. for (const auto &op_and_addr : saved_task_addrs_) {
  137. auto addr = op_and_addr.second;
  138. if (addr != nullptr) {
  139. GE_CHK_RT(rtFree(addr));
  140. }
  141. addr = nullptr;
  142. }
  143. saved_task_addrs_.clear();
  144. GE_CHK_STATUS(ModelRunStop());
  145. op_list_.clear();
  146. data_op_list_.clear();
  147. output_op_list_.clear();
  148. tensor_name_to_fixed_addr_size_.clear();
  149. tensor_name_to_peer_output_index_.clear();
  150. GE_DELETE_NEW_SINGLE(data_inputer_);
  151. // check rt ctx is exist. rt api call will cause error log when ctx not exist
  152. rtContext_t ctx = nullptr;
  153. rtError_t rt_ret = rtCtxGetCurrent(&ctx);
  154. if (rt_ret == RT_ERROR_NONE) {
  155. UnbindTaskSinkStream();
  156. for (size_t i = 0; i < label_list_.size(); ++i) {
  157. if (label_list_[i] != nullptr) {
  158. GE_LOGW_IF(rtLabelDestroy(label_list_[i]) != RT_ERROR_NONE, "Destroy label failed, index: %zu", i);
  159. }
  160. }
  161. for (size_t i = 0; i < stream_list_.size(); ++i) {
  162. GE_LOGW_IF(rtStreamDestroy(stream_list_[i]) != RT_ERROR_NONE, "Destroy stream failed, index: %zu", i);
  163. }
  164. for (size_t i = 0; i < event_list_.size(); ++i) {
  165. GE_LOGW_IF(rtEventDestroy(event_list_[i]) != RT_ERROR_NONE, "Destroy event failed, index: %zu", i);
  166. }
  167. FreeWeightsMem();
  168. FreeFeatureMapMem();
  169. FreeP2PMem();
  170. if (l1_fusion_addr_ != nullptr) {
  171. GE_CHK_RT(rtFree(l1_fusion_addr_));
  172. }
  173. if (rt_model_handle_ != nullptr) {
  174. GE_CHK_RT(rtModelDestroy(rt_model_handle_));
  175. rt_model_handle_ = nullptr;
  176. }
  177. }
  178. OpDebugUnRegister();
  179. GELOGI("do ReleaseTask");
  180. ReleaseTask();
  181. CleanTbeHandle();
  182. var_mem_base_ = nullptr;
  183. if (known_node_) {
  184. if (args_ != nullptr) {
  185. GE_CHK_RT(rtFree(args_));
  186. }
  187. total_io_addrs_.clear();
  188. if (fixed_addrs_ != nullptr) {
  189. GE_CHK_RT(rtFree(fixed_addrs_));
  190. }
  191. }
  192. } catch (...) {
  193. GELOGW("DavinciModel::~DavinciModel: clear op_list catch exception.");
  194. }
  195. }
  196. void DavinciModel::UnbindHcomStream() {
  197. if (!all_hccl_stream_list_.empty()) {
  198. for (size_t i = 0; i < all_hccl_stream_list_.size(); i++) {
  199. GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, all_hccl_stream_list_[i]) != RT_ERROR_NONE,
  200. "Unbind hccl stream from model failed! Index: %zu", i);
  201. GE_LOGW_IF(rtStreamDestroy(all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Destroy hccl stream for rt_model failed!")
  202. }
  203. }
  204. return;
  205. }
  206. void DavinciModel::ReleaseTask() {
  207. for (const auto &task : cpu_task_list_) {
  208. if (task != nullptr) {
  209. GE_CHK_STATUS(task->Release(), "Release task failed.");
  210. }
  211. }
  212. cpu_task_list_.clear();
  213. for (const auto &task : task_list_) {
  214. if (task != nullptr) {
  215. GE_CHK_STATUS(task->Release(), "Release task failed.");
  216. }
  217. }
  218. }
  219. Status DavinciModel::Assign(const GeModelPtr &ge_model) {
  220. if (ge_model == nullptr) {
  221. GELOGI("can't assign null ge_model");
  222. return FAILED;
  223. }
  224. ge_model_ = ge_model;
  225. return SUCCESS;
  226. }
  227. ///
  228. /// @ingroup ge
  229. /// @brief Reduce memory usage after task sink.
  230. /// @return: void
  231. ///
  232. void DavinciModel::Shrink() {
  233. ge_model_.reset(); // delete object.
  234. op_list_.clear();
  235. }
  236. Status DavinciModel::InitModelMem(void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) {
  237. if (is_model_has_inited_) {
  238. GELOGE(FAILED, "call InitModelMem more than once .");
  239. return FAILED;
  240. }
  241. is_model_has_inited_ = true;
  242. std::size_t data_size = TotalMemSize();
  243. std::size_t p2p_data_size = P2PMemInfos().at(RT_MEMORY_P2P_DDR).memory_size;
  244. const Buffer &weights = ge_model_->GetWeight();
  245. std::size_t weights_size = weights.GetSize();
  246. GE_CHECK_LE(weights_size, ALLOC_MEMORY_MAX_SIZE);
  247. if ((dev_ptr != nullptr) && (mem_size < TotalMemSize())) {
  248. GELOGE(FAILED, "Invalid mem param: mem_size=%zu totalsize=%zu.", mem_size, TotalMemSize());
  249. return FAILED;
  250. }
  251. if ((weight_ptr != nullptr) && (weight_size < weights_size)) {
  252. GELOGE(FAILED, "Invalid mem param: weight_size=%zu totalsize=%zu.", weight_size, weights_size);
  253. return FAILED;
  254. }
  255. mem_base_ = static_cast<uint8_t *>(dev_ptr);
  256. p2p_mem_base_ = static_cast<uint8_t *>(dev_ptr);
  257. weights_mem_base_ = static_cast<uint8_t *>(dev_ptr);
  258. is_inner_mem_base_ = false;
  259. is_inner_weight_base_ = false;
  260. if (TotalMemSize() && mem_base_ == nullptr) {
  261. mem_base_ = MallocFeatureMapMem(data_size);
  262. if (mem_base_ == nullptr) {
  263. GELOGE(GE_EXEC_ALLOC_FEATURE_MAP_MEM_FAILED, "Alloc feature map memory failed. size: %zu", data_size);
  264. return GE_EXEC_ALLOC_FEATURE_MAP_MEM_FAILED;
  265. }
  266. GEEVENT("[IMAS]InitModelMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id,
  267. mem_base_, data_size);
  268. weights_mem_base_ = mem_base_;
  269. is_inner_mem_base_ = true;
  270. is_inner_weight_base_ = true;
  271. }
  272. if (p2p_data_size != 0) {
  273. p2p_mem_base_ = MallocP2PMem(p2p_data_size);
  274. if (p2p_mem_base_ == nullptr) {
  275. GELOGE(GE_EXEC_ALLOC_P2P_MEM_FAILED, "Alloc p2p memory failed,size: %zu", p2p_data_size);
  276. return GE_EXEC_ALLOC_P2P_MEM_FAILED;
  277. }
  278. GELOGI("InitModelMem graph_%u MallocMemory type[P] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id,
  279. p2p_mem_base_, p2p_data_size);
  280. is_inner_p2p_mem_base_ = true;
  281. }
  282. if (weights_size != 0) {
  283. weights_mem_base_ = static_cast<uint8_t *>(weight_ptr);
  284. is_inner_weight_base_ = false;
  285. if (weight_ptr == nullptr) {
  286. weights_mem_base_ = MallocWeightsMem(weights_size);
  287. if (weights_mem_base_ == nullptr) {
  288. GELOGE(GE_EXEC_ALLOC_WEIGHT_MEM_FAILED, "Alloc weight memory failed. size: %zu", weights_size);
  289. return GE_EXEC_ALLOC_WEIGHT_MEM_FAILED;
  290. }
  291. is_inner_weight_base_ = true;
  292. }
  293. GELOGI("[IMAS]InitModelMem graph_%u MallocMemory type[W] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id,
  294. weights_mem_base_, weights_size);
  295. GE_CHK_RT_RET(rtMemcpy(weights_mem_base_, weights_size, weights.GetData(), weights_size, RT_MEMCPY_HOST_TO_DEVICE));
  296. GELOGI("copy weights data to device");
  297. }
  298. GE_CHK_STATUS_RET(InitVariableMem(), "Init variable memory failed.");
  299. runtime_param_.mem_base = mem_base_;
  300. runtime_param_.weight_base = weights_mem_base_;
  301. runtime_param_.memory_infos[RT_MEMORY_P2P_DDR].memory_base = p2p_mem_base_;
  302. return SUCCESS;
  303. }
  304. Status DavinciModel::InitVariableMem() {
  305. // malloc variable memory base
  306. var_mem_base_ = VarManager::Instance(session_id_)->GetVarMemoryBase(RT_MEMORY_HBM);
  307. if (TotalVarMemSize() && var_mem_base_ == nullptr) {
  308. Status ret = VarManager::Instance(session_id_)->MallocVarMemory(TotalVarMemSize());
  309. if (ret != SUCCESS) {
  310. GELOGE(ret, "Malloc variable memory failed.");
  311. return ret;
  312. }
  313. var_mem_base_ = VarManager::Instance(session_id_)->GetVarMemoryBase(RT_MEMORY_HBM);
  314. GEEVENT("[IMAS]InitVariableMem graph_%u MallocMemory type[V] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id,
  315. var_mem_base_, TotalVarMemSize());
  316. }
  317. runtime_param_.var_base = var_mem_base_;
  318. return SUCCESS;
  319. }
  320. void DavinciModel::InitRuntimeParams() {
  321. int64_t value = 0;
  322. bool ret;
  323. MemInfo p2p_mem_info;
  324. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_MEMORY_SIZE, value);
  325. runtime_param_.mem_size = ret ? (uint64_t)value : 0;
  326. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_WEIGHT_SIZE, value);
  327. runtime_param_.weight_size = ret ? (uint64_t)value : 0;
  328. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_STREAM_NUM, value);
  329. runtime_param_.stream_num = ret ? (uint32_t)value : 0;
  330. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_EVENT_NUM, value);
  331. runtime_param_.event_num = ret ? (uint32_t)value : 0;
  332. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_LABEL_NUM, value);
  333. runtime_param_.label_num = ret ? (uint32_t)value : 0;
  334. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_BATCH_NUM, value);
  335. runtime_param_.batch_num = ret ? (uint32_t)value : 0;
  336. ret = ge::AttrUtils::GetInt(ge_model_, MODEL_ATTR_TASK_GEN_BASE_ADDR, value);
  337. runtime_param_.logic_mem_base = ret ? (uint64_t)value : 0;
  338. ret = ge::AttrUtils::GetInt(ge_model_, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, value);
  339. runtime_param_.logic_weight_base = ret ? (uint64_t)value : 0;
  340. ret = ge::AttrUtils::GetInt(ge_model_, ge::MODEL_ATTR_SESSION_ID, value);
  341. runtime_param_.session_id = ret ? (uint64_t)value : 0;
  342. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_TASK_GEN_VAR_ADDR, value);
  343. runtime_param_.logic_var_base = ret ? (uint64_t)value : 0;
  344. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_VAR_SIZE, value);
  345. runtime_param_.var_size = ret ? (uint64_t)value : 0;
  346. session_id_ = runtime_param_.session_id;
  347. ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_P2P_MEMORY_SIZE, value);
  348. p2p_mem_info.memory_size = ret ? (uint64_t)value : 0;
  349. runtime_param_.memory_infos[RT_MEMORY_P2P_DDR] = std::move(p2p_mem_info);
  350. GELOGI(
  351. "InitRuntimeParams(), session_id:%lu, stream_num:%u, event_num:%u, label_num:%u, "
  352. "logic_mem_base:0x%lx, logic_weight_base:0x%lx, logic_var_base:0x%lx, "
  353. "memory_size:%lu, weight_size:%lu, var_size:%lu",
  354. runtime_param_.session_id, runtime_param_.stream_num, runtime_param_.event_num, runtime_param_.label_num,
  355. runtime_param_.logic_mem_base, runtime_param_.logic_weight_base, runtime_param_.logic_var_base,
  356. runtime_param_.mem_size, runtime_param_.weight_size, runtime_param_.var_size);
  357. }
  358. void DavinciModel::CheckHasHcomOp() {
  359. Graph graph = ge_model_->GetGraph();
  360. auto compute_graph = GraphUtils::GetComputeGraph(graph);
  361. if (compute_graph == nullptr) {
  362. return;
  363. }
  364. for (const auto &node : compute_graph->GetAllNodes()) {
  365. OpDescPtr op_desc = node->GetOpDesc();
  366. GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGW("Node OpDesc is nullptr"); continue);
  367. GE_IF_BOOL_EXEC(((op_desc->GetType() == HCOMBROADCAST) || (op_desc->GetType() == HCOMALLGATHER) ||
  368. (op_desc->GetType() == HCOMALLREDUCE) || (op_desc->GetType() == HCOMSEND) ||
  369. (op_desc->GetType() == HCOMRECEIVE) || (op_desc->GetType() == HCOMREDUCESCATTER) ||
  370. (op_desc->GetType() == HVDCALLBACKALLREDUCE) || (op_desc->GetType() == HVDCALLBACKALLGATHER) ||
  371. (op_desc->GetType() == HVDCALLBACKBROADCAST) || (op_desc->GetType() == HVDWAIT)),
  372. uint32_t stream_id = static_cast<uint32_t>(op_desc->GetStreamId());
  373. (void)hcom_streams_.emplace(stream_id); GELOGD("hcom stream: %u.", stream_id); continue);
  374. }
  375. }
  376. ///
  377. /// @ingroup ge
  378. /// @brief Make active stream list and bind to model.
  379. /// @return: 0 for success / others for fail
  380. ///
  381. Status DavinciModel::BindModelStream() {
  382. // Stream not in active_stream_indication_ is active stream.
  383. is_stream_list_bind_ = false;
  384. if ((!input_queue_ids_.empty() || !output_queue_ids_.empty()) || (deploy_type_ == AICPU_DEPLOY_CROSS_THREAD)) {
  385. for (size_t i = 0; i < stream_list_.size(); ++i) {
  386. if (active_stream_indication_.count(i) == 0) {
  387. active_stream_list_.push_back(stream_list_[i]);
  388. active_stream_indication_.insert(i); // deactive all model stream.
  389. }
  390. }
  391. }
  392. for (size_t i = 0; i < stream_list_.size(); ++i) {
  393. if (active_stream_indication_.count(i) > 0) {
  394. GELOGI("rtModelBindStream[%zu]", i);
  395. GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, stream_list_[i], RT_INVALID_FLAG));
  396. } else {
  397. // bind rt_model_handel to all streams that relates to op
  398. GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, stream_list_[i], RT_HEAD_STREAM));
  399. }
  400. }
  401. is_stream_list_bind_ = true;
  402. return SUCCESS;
  403. }
  404. Status DavinciModel::DoTaskSink() {
  405. // task sink is supported as model_task_def is set
  406. const auto &model_task_def = ge_model_->GetModelTaskDefPtr();
  407. if (model_task_def == nullptr) {
  408. return SUCCESS;
  409. }
  410. GE_CHK_RT_RET(rtGetAicpuDeploy(&deploy_type_));
  411. GELOGI("do task_sink. AiCpu deploy type is: %x.", deploy_type_);
  412. GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed.");
  413. if (known_node_) {
  414. GE_CHK_STATUS_RET(MallocKnownArgs(), "Mallloc known node args failed.");
  415. }
  416. GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "InitTaskInfo failed.");
  417. GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed.");
  418. GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed.");
  419. GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed.");
  420. GE_CHK_RT_RET(rtModelLoadComplete(rt_model_handle_));
  421. SetCopyOnlyOutput();
  422. return SUCCESS;
  423. }
  424. // set device use aicore(0) or vectorcore(1)
  425. Status DavinciModel::SetTSDevice() {
  426. int64_t value = 0;
  427. bool ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_CORE_TYPE, value);
  428. uint32_t core_type = ret ? static_cast<uint32_t>(value) : 0;
  429. GELOGI("SetTSDevice: %u", core_type);
  430. rtError_t rt_ret = rtSetTSDevice(core_type);
  431. if (rt_ret != RT_ERROR_NONE) {
  432. GELOGE(RT_FAILED, "SetTSDevice failed, ret: 0x%X", rt_ret);
  433. return RT_ERROR_TO_GE_STATUS(rt_ret);
  434. }
  435. return SUCCESS;
  436. }
  437. Status DavinciModel::OpDebugRegister() {
  438. bool is_op_debug = false;
  439. (void)ge::AttrUtils::GetBool(ge_model_, ATTR_OP_DEBUG_FLAG, is_op_debug);
  440. GELOGD("The value of op_debug in ge_model_ is %d.", is_op_debug);
  441. if (is_op_debug) {
  442. debug_reg_mutex_.lock();
  443. rtError_t rt_ret = rtMalloc(&op_debug_addr_, kOpDebugMemorySize, RT_MEMORY_DDR);
  444. if (rt_ret != RT_ERROR_NONE) {
  445. GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret);
  446. return RT_ERROR_TO_GE_STATUS(rt_ret);
  447. }
  448. uint64_t debug_addrs_tmp = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(op_debug_addr_));
  449. // For data dump, aicpu needs the pointer to pointer that save the real debug address.
  450. rt_ret = rtMalloc(&p2p_debug_addr_, kDebugP2pSize, RT_MEMORY_HBM);
  451. if (rt_ret != RT_ERROR_NONE) {
  452. GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret);
  453. return RT_ERROR_TO_GE_STATUS(rt_ret);
  454. }
  455. rt_ret = rtMemcpy(p2p_debug_addr_, sizeof(uint64_t), &debug_addrs_tmp, sizeof(uint64_t), RT_MEMCPY_HOST_TO_DEVICE);
  456. if (rt_ret != RT_ERROR_NONE) {
  457. GELOGE(RT_FAILED, "rtMemcpy to p2p_addr error: 0x%X", rt_ret);
  458. return RT_ERROR_TO_GE_STATUS(rt_ret);
  459. }
  460. uint32_t op_debug_mode = 0;
  461. (void)ge::AttrUtils::GetInt(ge_model_, ATTR_OP_DEBUG_MODE, op_debug_mode);
  462. GELOGD("The value of op_debug_mode in ge_model_ is %u.", op_debug_mode);
  463. uint32_t debug_task_id = 0;
  464. uint32_t debug_stream_id = 0;
  465. rt_ret = rtDebugRegister(rt_model_handle_, op_debug_mode, op_debug_addr_, &debug_stream_id, &debug_task_id);
  466. if (rt_ret != RT_ERROR_NONE) {
  467. GELOGE(RT_FAILED, "rtDebugRegister error, ret: 0x%X", rt_ret);
  468. return RT_ERROR_TO_GE_STATUS(rt_ret);
  469. }
  470. GELOGI("debug_task_id:%d, debug_stream_id:%u", debug_task_id, debug_stream_id);
  471. is_op_debug_reg_ = true;
  472. data_dumper_.SaveOpDebugId(debug_task_id, debug_stream_id, p2p_debug_addr_, is_op_debug);
  473. }
  474. return SUCCESS;
  475. }
  476. void DavinciModel::OpDebugUnRegister() {
  477. GELOGI("OpDebugUnRegister, is_op_debug_reg_ = %d", is_op_debug_reg_);
  478. if (is_op_debug_reg_) {
  479. debug_reg_mutex_.unlock();
  480. rtError_t rt_ret = RT_ERROR_NONE;
  481. if (rt_model_handle_ != nullptr) {
  482. GELOGD("start call debug_unregister.");
  483. rt_ret = rtDebugUnRegister(rt_model_handle_);
  484. if (rt_ret != RT_ERROR_NONE) {
  485. GELOGW("rtDebugUnRegister failed, ret: 0x%X", rt_ret);
  486. }
  487. }
  488. if (op_debug_addr_ != nullptr) {
  489. rt_ret = rtFree(op_debug_addr_);
  490. if (rt_ret != RT_ERROR_NONE) {
  491. GELOGW("rtFree failed, ret: 0x%X", rt_ret);
  492. }
  493. op_debug_addr_ = nullptr;
  494. }
  495. if (p2p_debug_addr_ != nullptr) {
  496. rt_ret = rtFree(p2p_debug_addr_);
  497. if (rt_ret != RT_ERROR_NONE) {
  498. GELOGW("rtFree failed, ret: 0x%X", rt_ret);
  499. }
  500. p2p_debug_addr_ = nullptr;
  501. }
  502. is_op_debug_reg_ = false;
  503. }
  504. return;
  505. }
  506. // initialize op sequence and call initialization function of each op respectively
  507. Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) {
  508. // validating params
  509. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(priority_ < 0 || priority_ > 7, return PARAM_INVALID,
  510. "Priority must between 0-7, now is %d", priority_);
  511. GE_CHK_BOOL_RET_STATUS(ge_model_ != nullptr, PARAM_INVALID, "GeModel is null.");
  512. Graph graph = ge_model_->GetGraph();
  513. ComputeGraphPtr compute_graph = GraphUtils::GetComputeGraph(graph);
  514. GE_CHK_BOOL_RET_STATUS(compute_graph != nullptr, INTERNAL_ERROR, "Get compute graph is nullptr.");
  515. // Initializing runtime_param_
  516. InitRuntimeParams();
  517. // RTS set aicore or vectorcore
  518. GE_CHK_STATUS_RET(SetTSDevice(), "SetTSDevice failed");
  519. version_ = ge_model_->GetVersion();
  520. name_ = ge_model_->GetName();
  521. (void)ge::AttrUtils::GetBool(ge_model_, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_);
  522. GELOGD("The value of ge.l1Fusion in ge_model_ is %d.", is_l1_fusion_enable_);
  523. CheckHasHcomOp();
  524. vector<int64_t> huge_stream_list;
  525. (void)ge::AttrUtils::GetListInt(ge_model_, ATTR_MODEL_HUGE_STREAM_LIST, huge_stream_list);
  526. std::set<int64_t> huge_streams(huge_stream_list.begin(), huge_stream_list.end());
  527. for (uint32_t i = 0; i < StreamNum(); i++) {
  528. rtStream_t stream = nullptr;
  529. GE_MAKE_GUARD_RTSTREAM(stream);
  530. uint32_t stream_flags = RT_STREAM_PERSISTENT;
  531. if (huge_streams.find(i) != huge_streams.end()) {
  532. GELOGI("Stream %u is huge stream.", i);
  533. stream_flags |= RT_STREAM_HUGE;
  534. }
  535. if (hcom_streams_.find(i) != hcom_streams_.end()) {
  536. GE_CHK_RT_RET(rtStreamCreateWithFlags(&stream, priority_, stream_flags | RT_STREAM_FORCE_COPY));
  537. } else {
  538. GE_CHK_RT_RET(rtStreamCreateWithFlags(&stream, priority_, stream_flags));
  539. }
  540. GE_DISMISS_GUARD(stream);
  541. stream_list_.push_back(stream);
  542. GELOGD("Stream index:%u, stream:%p.", i, stream);
  543. }
  544. for (uint32_t i = 0; i < EventNum(); i++) {
  545. rtEvent_t rt_event;
  546. GE_CHK_RT_RET(rtEventCreate(&rt_event));
  547. event_list_.push_back(rt_event);
  548. }
  549. label_list_.resize(LabelNum(), nullptr);
  550. // create model_handle to load model
  551. GE_CHK_RT_RET(rtModelCreate(&rt_model_handle_, 0));
  552. GE_CHK_RT_RET(rtModelGetId(rt_model_handle_, &runtime_model_id_));
  553. // inference will use default graph_id 0;
  554. runtime_param_.graph_id = compute_graph->GetGraphID();
  555. // op debug register
  556. GE_CHK_STATUS_RET(OpDebugRegister(), "OpDebugRegister failed");
  557. GE_TIMESTAMP_START(TransAllVarData);
  558. GE_CHK_STATUS_RET(TransAllVarData(compute_graph, runtime_param_.graph_id), "TransAllVarData failed.");
  559. GE_TIMESTAMP_END(TransAllVarData, "GraphLoader::TransAllVarData");
  560. GE_CHK_STATUS_RET(TransVarDataUtils::CopyVarData(compute_graph, session_id_, device_id_), "copy var data failed.");
  561. GE_TIMESTAMP_START(InitModelMem);
  562. GELOGI("Known node is %d", known_node_);
  563. if (!known_node_) {
  564. GE_CHK_STATUS_RET_NOLOG(InitModelMem(dev_ptr, mem_size, weight_ptr, weight_size));
  565. data_inputer_ = new (std::nothrow) DataInputer();
  566. GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, MEMALLOC_FAILED, "data_inputer_ is nullptr.");
  567. }
  568. GE_TIMESTAMP_END(InitModelMem, "GraphLoader::InitModelMem");
  569. for (const ge::NodePtr &node : compute_graph->GetDirectNode()) {
  570. auto op_desc = node->GetOpDesc();
  571. GE_IF_BOOL_EXEC(op_desc == nullptr, continue);
  572. GetFixedAddrAttr(op_desc);
  573. GE_IF_BOOL_EXEC(op_desc->GetType() != VARIABLE, continue);
  574. GE_IF_BOOL_EXEC(IsBroadCastOpData(node),
  575. (void)ge::AttrUtils::SetStr(op_desc, VAR_ATTR_VAR_IS_BROADCAST, "var_is_restore"););
  576. }
  577. // for profiling
  578. op_name_map_ = compute_graph->GetGraphOpName();
  579. vector<string> op_name;
  580. GE_IF_BOOL_EXEC(ge::AttrUtils::GetListStr(ge_model_, ATTR_MODEL_TASK_INDEX_OP_NAME, op_name),
  581. GELOGI("get str of task_index_op_name"));
  582. if (op_name_map_.empty()) {
  583. for (size_t idx = 0; idx < op_name.size(); idx++) {
  584. op_name_map_[idx] = op_name[idx];
  585. }
  586. GELOGI("Infer profiling: op_name_size(%zu)", op_name.size());
  587. }
  588. GE_CHK_STATUS_RET(InitNodes(compute_graph), "Init nodes failed");
  589. SetDataDumperArgs(compute_graph);
  590. GE_TIMESTAMP_START(DoTaskSink);
  591. auto ret = DoTaskSink();
  592. GE_TIMESTAMP_END(DoTaskSink, "GraphLoader::DoTaskSink");
  593. auto all_dump_model = GetDumpProperties().GetAllDumpModel();
  594. bool findByOmName = all_dump_model.find(om_name_) != all_dump_model.end();
  595. bool findByModelName = all_dump_model.find(name_) != all_dump_model.end();
  596. if (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end() || findByOmName || findByModelName) {
  597. // malloc 2M for dump l1fusion op
  598. GE_CHK_RT_RET(rtMalloc(&l1_fusion_addr_, kDumpL1FusionOpMByteSize, RT_MEMORY_DDR));
  599. // send l1fusion dump addr to rts
  600. GE_CHK_RT_RET(rtDumpAddrSet(rt_model_handle_, l1_fusion_addr_, kDumpL1FusionOpMByteSize, kDumpFlagOfL1Fusion));
  601. }
  602. /// In zero copy model, if a aicpu operator is connected to the first or last layer, before model execution,
  603. /// the aicpu opertor needs to destroy history record, and update operator memory address.
  604. /// The model with specified aicpu operators is only marked here, and destruction is in ModelManager::ExecuteModel().
  605. need_destroy_aicpu_kernel_ = IsAicpuKernelConnectSpecifiedLayer();
  606. (void)ge::AttrUtils::GetListStr(ge_model_, ATTR_MODEL_OUT_NODES_NAME, out_node_name_);
  607. // collect profiling for ge
  608. if (ProfilingManager::Instance().ProfilingModelLoadOn()) {
  609. std::vector<ComputeGraphDescInfo> compute_graph_desc_info;
  610. Status ret1 = GetComputeGraphInfo(compute_graph, compute_graph_desc_info);
  611. if (ret1 != SUCCESS) {
  612. GELOGE(ret1, "GetComputeGraphInfo failed.");
  613. return ret1;
  614. }
  615. ProfilingManager::Instance().ReportProfilingData(GetTaskDescInfo(), compute_graph_desc_info);
  616. GE_CHK_STATUS(SinkModelProfile(), "Sink model profile failed.");
  617. }
  618. Shrink();
  619. GELOGI("Davinci model init success.");
  620. return ret;
  621. }
  622. ///
  623. /// @ingroup ge
  624. /// @brief Travel all nodes and determine if destruction is required.
  625. /// @return bool
  626. ///
  627. bool DavinciModel::IsAicpuKernelConnectSpecifiedLayer() {
  628. Graph graph = ge_model_->GetGraph();
  629. ComputeGraphPtr compute_graph = GraphUtils::GetComputeGraph(graph);
  630. auto all_nodes = compute_graph->GetAllNodes();
  631. for (auto &node : all_nodes) {
  632. GE_IF_BOOL_EXEC(node == nullptr, continue);
  633. OpDescPtr op_desc = node->GetOpDesc();
  634. GE_IF_BOOL_EXEC(op_desc == nullptr, continue);
  635. int64_t imply_type = -1;
  636. (void)ge::AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, imply_type);
  637. if (imply_type != static_cast<int64_t>(domi::ImplyType::AI_CPU)) {
  638. continue;
  639. }
  640. GELOGD("Current operator imply type is %ld, name is %s.", imply_type, op_desc->GetName().c_str());
  641. for (auto &in_data_anchor : node->GetAllInDataAnchors()) {
  642. GE_IF_BOOL_EXEC(in_data_anchor == nullptr, continue);
  643. auto peer_out_data_anchor = in_data_anchor->GetPeerOutAnchor();
  644. GE_IF_BOOL_EXEC(peer_out_data_anchor == nullptr, continue);
  645. auto peer_node = peer_out_data_anchor->GetOwnerNode();
  646. GE_IF_BOOL_EXEC(peer_node == nullptr, continue);
  647. auto peer_op_desc = peer_node->GetOpDesc();
  648. GE_IF_BOOL_EXEC(peer_op_desc == nullptr, continue);
  649. if (IsDataOp(peer_op_desc->GetType())) {
  650. GELOGI("Mark specified aicpu operator connected to data.");
  651. return true;
  652. }
  653. }
  654. for (auto &out_data_anchor : node->GetAllOutDataAnchors()) {
  655. GE_IF_BOOL_EXEC(out_data_anchor == nullptr, continue);
  656. auto peer_in_data_anchors = out_data_anchor->GetPeerInDataAnchors();
  657. for (auto &peer_in_data_anchor : peer_in_data_anchors) {
  658. GE_IF_BOOL_EXEC(peer_in_data_anchor == nullptr, continue);
  659. auto peer_node = peer_in_data_anchor->GetOwnerNode();
  660. GE_IF_BOOL_EXEC(peer_node == nullptr, continue);
  661. auto peer_op_desc = peer_node->GetOpDesc();
  662. GE_IF_BOOL_EXEC(peer_op_desc == nullptr, continue);
  663. if (peer_op_desc->GetType() == NETOUTPUT) {
  664. GELOGI("Mark specified aicpu operator connected to netoutput.");
  665. return true;
  666. }
  667. }
  668. }
  669. }
  670. return false;
  671. }
  672. Status DavinciModel::UpdateSessionId(uint64_t session_id) {
  673. GE_CHECK_NOTNULL(ge_model_);
  674. if (!AttrUtils::SetInt(ge_model_, MODEL_ATTR_SESSION_ID, static_cast<int64_t>(session_id))) {
  675. GELOGW("Set attr[%s] failed in updating session_id.", MODEL_ATTR_SESSION_ID.c_str());
  676. }
  677. GELOGD("Update session id: %lu.", session_id);
  678. return SUCCESS;
  679. }
  680. ///
  681. /// @ingroup ge
  682. /// @brief Travel all nodes and do some init.
  683. /// @param [in] compute_graph: ComputeGraph to load.
  684. /// @return Status
  685. ///
  686. Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) {
  687. uint32_t data_op_index = 0;
  688. GE_TIMESTAMP_CALLNUM_START(LoadTBEKernelBinToOpDesc);
  689. GE_TIMESTAMP_CALLNUM_START(InitTbeHandle);
  690. typedef Status (DavinciModel::*OpDescCall)(const OpDescPtr &);
  691. static std::map<std::string, OpDescCall> op_desc_handle = {
  692. {VARIABLE, &DavinciModel::InitVariable},
  693. {CONSTANTOP, &DavinciModel::InitConstant},
  694. {STREAMACTIVE, &DavinciModel::InitStreamActive},
  695. {STREAMSWITCH, &DavinciModel::InitStreamSwitch},
  696. {STREAMSWITCHN, &DavinciModel::InitStreamSwitchN},
  697. {LABELSET, &DavinciModel::InitLabelSet},
  698. {CASE, &DavinciModel::InitCase},
  699. };
  700. GE_CHK_STATUS_RET(InitInputOutputForDynamic(compute_graph), "InitInputOutputForDynamic failed.");
  701. map<uint32_t, OpDescPtr> data_by_index;
  702. auto nodes = compute_graph->GetAllNodes();
  703. const CustAICPUKernelStore &aicpu_kernel_store = ge_model_->GetCustAICPUKernelStore();
  704. for (size_t i = 0; i < nodes.size(); i++) {
  705. auto node = nodes.at(i);
  706. auto op_desc = node->GetOpDesc();
  707. if (op_desc == nullptr) {
  708. GELOGE(PARAM_INVALID, "op_desc is null.");
  709. return PARAM_INVALID;
  710. }
  711. op_list_[op_desc->GetId()] = op_desc;
  712. GE_TIMESTAMP_RESTART(LoadTBEKernelBinToOpDesc);
  713. aicpu_kernel_store.LoadCustAICPUKernelBinToOpDesc(op_desc);
  714. GE_TIMESTAMP_ADD(LoadTBEKernelBinToOpDesc);
  715. if (IsDataOp(op_desc->GetType())) {
  716. if (InitDataOp(node, data_op_index, data_by_index) != SUCCESS) {
  717. GELOGE(PARAM_INVALID, "Data init failed, Name: %s", op_desc->GetName().c_str());
  718. return PARAM_INVALID;
  719. }
  720. data_dumper_.SaveDumpInput(node);
  721. continue;
  722. }
  723. if (op_desc->GetType() == NETOUTPUT) {
  724. if (InitNetOutput(node) != SUCCESS) {
  725. GELOGE(PARAM_INVALID, "NetOutput init failed, Name: %s", op_desc->GetName().c_str());
  726. return PARAM_INVALID;
  727. }
  728. continue;
  729. }
  730. auto it = op_desc_handle.find(op_desc->GetType());
  731. if (it != op_desc_handle.end()) {
  732. if ((this->*it->second)(op_desc) != SUCCESS) {
  733. GELOGE(PARAM_INVALID, "NetOutput init failed, Name: %s", op_desc->GetName().c_str());
  734. return PARAM_INVALID;
  735. }
  736. continue;
  737. }
  738. if (IsNoTaskAndDumpNeeded(op_desc)) {
  739. GELOGD("node[%s] without task, and save op_desc and addr for dump", op_desc->GetName().c_str());
  740. const RuntimeParam &rts_param = GetRuntimeParam();
  741. const vector<void *> input_data_addrs = ModelUtils::GetInputDataAddrs(rts_param, op_desc);
  742. const vector<void *> output_data_addrs = ModelUtils::GetOutputDataAddrs(rts_param, op_desc);
  743. const vector<void *> workspace_data_addrs = ModelUtils::GetWorkspaceDataAddrs(rts_param, op_desc);
  744. vector<void *> tensor_device_addrs;
  745. tensor_device_addrs.insert(tensor_device_addrs.end(), input_data_addrs.begin(), input_data_addrs.end());
  746. tensor_device_addrs.insert(tensor_device_addrs.end(), output_data_addrs.begin(), output_data_addrs.end());
  747. tensor_device_addrs.insert(tensor_device_addrs.end(), workspace_data_addrs.begin(), workspace_data_addrs.end());
  748. void *addr = nullptr;
  749. auto size = kAddrLen * tensor_device_addrs.size();
  750. GE_CHK_RT_RET(rtMalloc(&addr, size, RT_MEMORY_HBM));
  751. rtError_t rt_ret = rtMemcpy(addr, size, tensor_device_addrs.data(), size, RT_MEMCPY_HOST_TO_DEVICE);
  752. if (rt_ret != RT_ERROR_NONE) {
  753. GELOGE(RT_FAILED, "rtMemcpy error, ret: 0x%X", rt_ret);
  754. GE_CHK_RT(rtFree(addr));
  755. return RT_ERROR_TO_GE_STATUS(rt_ret);
  756. }
  757. saved_task_addrs_.emplace(op_desc, addr);
  758. }
  759. GE_TIMESTAMP_RESTART(InitTbeHandle);
  760. uint32_t run_mode = static_cast<uint32_t>(domi::ImplyType::INVALID);
  761. if (AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, run_mode) &&
  762. run_mode == static_cast<uint32_t>(domi::ImplyType::TVM)) {
  763. // Skip no_task operator, such as concat and split.
  764. bool attr_notask = false;
  765. bool get_attr_notask_flag = ge::AttrUtils::GetBool(op_desc, ATTR_NAME_NOTASK, attr_notask);
  766. GE_IF_BOOL_EXEC(get_attr_notask_flag && attr_notask,
  767. GELOGI("Node[name:%s, type:%s] does not generate task, skip initialization.",
  768. op_desc->GetName().c_str(), op_desc->GetType().c_str());
  769. continue;);
  770. Status status = InitTbeHandle(op_desc);
  771. if (status != SUCCESS) {
  772. GELOGE(status, "TBE init failed. %s", op_desc->GetName().c_str());
  773. return status;
  774. }
  775. }
  776. GE_TIMESTAMP_ADD(InitTbeHandle);
  777. }
  778. AdjustDataOpList(data_by_index);
  779. GE_TIMESTAMP_CALLNUM_END(LoadTBEKernelBinToOpDesc, "GraphLoader::LoadTBEKernelBinToOpDesc.");
  780. GE_TIMESTAMP_CALLNUM_END(InitTbeHandle, "GraphLoader::InitTbeHandle.");
  781. return SUCCESS;
  782. }
  783. Status DavinciModel::InitInputOutputForDynamic(const ComputeGraphPtr &compute_graph) {
  784. if (!known_node_) return SUCCESS;
  785. // for dynamic shape
  786. auto direct_nodes = compute_graph->GetDirectNode();
  787. for (size_t i = 0; i < direct_nodes.size(); i++) {
  788. auto node = direct_nodes.at(i);
  789. auto op_desc = node->GetOpDesc();
  790. if (op_desc == nullptr) {
  791. GELOGE(PARAM_INVALID, "op_desc is null.");
  792. return PARAM_INVALID;
  793. }
  794. if (IsDataOp(op_desc->GetType())) {
  795. GELOGD("init data op %s", op_desc->GetName().c_str());
  796. data_op_list_.push_back(op_desc);
  797. }
  798. if (op_desc->GetType() == NETOUTPUT) {
  799. GELOGD("init netouput op %s", op_desc->GetName().c_str());
  800. output_op_list_.push_back(op_desc);
  801. }
  802. }
  803. return SUCCESS;
  804. }
  805. /// @ingroup ge
  806. /// @brief Data Op Initialize.
  807. /// @param [in] NodePtr: Data Op.
  808. /// @param [in/out] data_op_index: NetOutput addr size info.
  809. /// @return Status
  810. Status DavinciModel::InitDataOp(const NodePtr &node, uint32_t &data_op_index, map<uint32_t, OpDescPtr> &data_by_index) {
  811. // op_desc Checked by Init: Data, valid.
  812. auto op_desc = node->GetOpDesc();
  813. if (known_node_) {
  814. return SUCCESS;
  815. }
  816. uint32_t parent_index = 0; // Ignore subgraph Data Node.
  817. if (AttrUtils::GetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
  818. GELOGI("Init zero copy by subgraph Data node: %s.", op_desc->GetName().c_str());
  819. return InitInputBatchLabel(node);
  820. }
  821. data_op_list_.push_back(op_desc);
  822. // Make information for copy input data.
  823. const vector<int64_t> output_size_list = ModelUtils::GetOutputSize(op_desc);
  824. const vector<void *> virtual_addr_list = ModelUtils::GetOutputDataAddrs(runtime_param_, op_desc);
  825. const vector<int64_t> output_offset_list = op_desc->GetOutputOffset();
  826. if (output_offset_list.size() != virtual_addr_list.size()) {
  827. GELOGE(PARAM_INVALID, "virtual_addr size:%zu should be equal to offset size:%zu.", virtual_addr_list.size(),
  828. output_offset_list.size());
  829. return PARAM_INVALID;
  830. }
  831. auto data_index = data_op_index;
  832. if (AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, data_index)) {
  833. GELOGI("ge_train: get new index %u, old %u", data_index, data_op_index);
  834. }
  835. bool fusion_flag = false;
  836. ZeroCopyOffset zero_copy_offset;
  837. Status ret = zero_copy_offset.InitInputDataInfo(output_size_list, virtual_addr_list, op_desc, fusion_flag);
  838. if (ret != SUCCESS) {
  839. GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", op_desc->GetName().c_str());
  840. return PARAM_INVALID;
  841. }
  842. new_input_data_info_[data_index] = zero_copy_offset;
  843. data_by_index[data_index] = op_desc;
  844. for (size_t index = 0; index < virtual_addr_list.size(); ++index) {
  845. void *addr = virtual_addr_list.at(index);
  846. if (new_input_outside_addrs_.find(addr) != new_input_outside_addrs_.end()) {
  847. continue;
  848. }
  849. zero_copy_offset.SetInputOutsideAddrs(output_offset_list, addr, index, fusion_flag, real_virtual_addrs_);
  850. new_input_outside_addrs_[addr] = zero_copy_offset;
  851. }
  852. GELOGI("SetInputOutsideAddr success.");
  853. data_op_index++;
  854. if (InitInputZeroCopyNodes(node) != SUCCESS) {
  855. GELOGE(PARAM_INVALID, "Input zero copy nodes init failed!");
  856. return PARAM_INVALID;
  857. }
  858. return SUCCESS;
  859. }
  860. ///
  861. /// @ingroup ge
  862. /// @brief Sort Data op list by index.
  863. /// @param [in] data_by_index: map of Data Op.
  864. /// @return
  865. ///
  866. void DavinciModel::AdjustDataOpList(const map<uint32_t, OpDescPtr> &data_by_index) {
  867. if (data_by_index.size() != data_op_list_.size()) {
  868. GELOGW("Data map size: %zu, Data list size: %zu.", data_by_index.size(), data_op_list_.size());
  869. return;
  870. }
  871. data_op_list_.clear();
  872. for (auto &item : data_by_index) {
  873. data_op_list_.emplace_back(item.second);
  874. }
  875. }
  876. ///
  877. /// @ingroup ge
  878. /// @brief input zero copy node Initialize.
  879. /// @param [in] NodePtr: Data Op.
  880. /// @return Status
  881. ///
  882. Status DavinciModel::InitInputZeroCopyNodes(const NodePtr &node) {
  883. auto out_data_anchor = node->GetOutDataAnchor(kDataIndex);
  884. if (out_data_anchor == nullptr) {
  885. GELOGE(FAILED, "Out data anchor is nullptr");
  886. return FAILED;
  887. }
  888. for (auto &peer_in_data_anchor : out_data_anchor->GetPeerInDataAnchors()) {
  889. auto node = peer_in_data_anchor->GetOwnerNode();
  890. auto op_desc = node->GetOpDesc();
  891. if (op_desc == nullptr) {
  892. GELOGE(FAILED, "Op desc is nullptr");
  893. return FAILED;
  894. }
  895. string batch_label;
  896. (void)ge::AttrUtils::GetStr(op_desc, ATTR_NAME_BATCH_LABEL, batch_label);
  897. if (batch_label.empty()) {
  898. batch_label = kDefaultBatchLable;
  899. }
  900. if (zero_copy_op_id_batch_label_.find(op_desc->GetId()) == zero_copy_op_id_batch_label_.end()) {
  901. zero_copy_op_id_batch_label_.emplace(pair<int64_t, string>(op_desc->GetId(), batch_label));
  902. GELOGD("Init input zero copy nodes success, op name:%s, op id: %ld, batch label: %s.", op_desc->GetName().c_str(),
  903. op_desc->GetId(), batch_label.c_str());
  904. }
  905. }
  906. return SUCCESS;
  907. }
  908. /// @ingroup ge
  909. /// @brief NetOutput Op Initialize.
  910. /// @param [in] NodePtr: NetOutput Op.
  911. /// @return Status
  912. Status DavinciModel::InitNetOutput(const NodePtr &node) {
  913. // node->GetOpDesc Checked by Init: NetOutput, valid.
  914. auto op_desc = node->GetOpDesc();
  915. // excludes the function op sub graph, e.g. case,if
  916. if (known_node_) {
  917. return SUCCESS;
  918. }
  919. ComputeGraphPtr owner_graph = node->GetOwnerComputeGraph();
  920. GE_CHECK_NOTNULL(owner_graph);
  921. if (owner_graph->GetParentGraph() != nullptr) {
  922. GELOGI("Init zero copy by subgraph NetOutput node: %s.", op_desc->GetName().c_str());
  923. op_list_.erase(op_desc->GetId());
  924. return InitOutputBatchLabel(node);
  925. }
  926. output_op_list_.push_back(op_desc);
  927. // Make information for copy output data.
  928. const vector<int64_t> input_size_list = ModelUtils::GetInputSize(op_desc);
  929. const vector<void *> virtual_addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc);
  930. const vector<int64_t> input_offset_list = op_desc->GetInputOffset();
  931. if (input_offset_list.size() != virtual_addr_list.size()) {
  932. GELOGE(PARAM_INVALID, "virtual_addr size should be equal to offset size.");
  933. return PARAM_INVALID;
  934. }
  935. if (input_size_list.empty() && virtual_addr_list.empty()) {
  936. GELOGI("NetOutput[%s] is empty.", op_desc->GetName().c_str());
  937. return SUCCESS;
  938. }
  939. if (input_size_list.empty() || input_size_list.size() != virtual_addr_list.size()) {
  940. GELOGE(PARAM_INVALID, "NetOutput[%s] init failed: Input size is %zu, Input addr is %zu", op_desc->GetName().c_str(),
  941. input_size_list.size(), virtual_addr_list.size());
  942. return PARAM_INVALID;
  943. }
  944. size_t num = new_output_data_info_.size();
  945. bool fusion_flag = false;
  946. for (size_t idx = 0; idx < input_size_list.size(); ++idx) {
  947. ZeroCopyOffset zero_copy_offset;
  948. Status ret = zero_copy_offset.InitOutputDataInfo(input_size_list, virtual_addr_list, op_desc, idx, fusion_flag);
  949. if (ret != SUCCESS) {
  950. GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", op_desc->GetName().c_str());
  951. return PARAM_INVALID;
  952. }
  953. new_output_data_info_[num + idx] = zero_copy_offset;
  954. void *addr = virtual_addr_list.at(idx);
  955. int64_t input_offset = input_offset_list.at(idx);
  956. vector<void *> tensor_addrs;
  957. zero_copy_offset.SetOutputOutsideAddrs(input_offset, fusion_flag, addr, tensor_addrs);
  958. auto rslt = new_output_outside_addrs_.insert(std::pair<void *, ZeroCopyOffset>(addr, zero_copy_offset));
  959. if (!rslt.second) {
  960. GELOGI("same output_tensor_addr %p to different input_tensor of %s", addr, op_desc->GetName().c_str());
  961. DisableZeroCopy(addr);
  962. }
  963. for (size_t i = 0; i < tensor_addrs.size(); ++i) {
  964. void *real_addr = tensor_addrs.at(i);
  965. DisableZeroCopy(real_addr);
  966. real_virtual_addrs_.insert(real_addr);
  967. }
  968. GELOGI("SetOutputOutsideAddr success.");
  969. }
  970. if (InitOutputZeroCopyNodes(node) != SUCCESS) {
  971. GELOGE(PARAM_INVALID, "Output zero copy nodes init failed!");
  972. return PARAM_INVALID;
  973. }
  974. GELOGI("DavinciModel::InitNetoutput success.");
  975. return SUCCESS;
  976. }
  977. ///
  978. /// @ingroup ge
  979. /// @brief output zero copy node Initialize.
  980. /// @param [in] NodePtr: netoutput Op.
  981. /// @return Status
  982. ///
  983. Status DavinciModel::InitOutputZeroCopyNodes(const NodePtr &node) {
  984. set<NodePtr> nodes_need_record;
  985. for (auto &in_data_anchor : node->GetAllInDataAnchors()) {
  986. auto peer_out_data_anchor = in_data_anchor->GetPeerOutAnchor();
  987. if (peer_out_data_anchor == nullptr) {
  988. continue;
  989. }
  990. auto peer_node = peer_out_data_anchor->GetOwnerNode();
  991. nodes_need_record.emplace(peer_node);
  992. // Merge node output multiplexed input, upstream nodes need to be considered in multiple batch scenarios
  993. if (peer_node->GetType() == MERGE) {
  994. for (const auto &merge_peer_in_data_anchor : peer_node->GetAllInDataAnchors()) {
  995. auto merge_peer_out_data_anchor = merge_peer_in_data_anchor->GetPeerOutAnchor();
  996. if (merge_peer_out_data_anchor == nullptr) {
  997. continue;
  998. }
  999. auto merge_peer_node = merge_peer_out_data_anchor->GetOwnerNode();
  1000. nodes_need_record.emplace(merge_peer_node);
  1001. }
  1002. } else {
  1003. for (const auto &other_in_data_anchor : peer_out_data_anchor->GetPeerInDataAnchors()) {
  1004. auto other_in_node = other_in_data_anchor->GetOwnerNode();
  1005. if (other_in_node->GetType() != NETOUTPUT) {
  1006. nodes_need_record.emplace(other_in_node);
  1007. }
  1008. }
  1009. }
  1010. }
  1011. for (const auto &node_need_record : nodes_need_record) {
  1012. auto op_desc = node_need_record->GetOpDesc();
  1013. GE_CHECK_NOTNULL(op_desc);
  1014. string batch_label;
  1015. (void)ge::AttrUtils::GetStr(op_desc, ATTR_NAME_BATCH_LABEL, batch_label);
  1016. if (batch_label.empty()) {
  1017. batch_label = kDefaultBatchLable;
  1018. }
  1019. if (zero_copy_op_id_batch_label_.find(op_desc->GetId()) == zero_copy_op_id_batch_label_.end()) {
  1020. zero_copy_op_id_batch_label_.emplace(pair<int64_t, string>(op_desc->GetId(), batch_label));
  1021. GELOGD("Init Output zero copy nodes success, op name:%s, op id: %ld, batch label: %s.",
  1022. op_desc->GetName().c_str(), op_desc->GetId(), batch_label.c_str());
  1023. }
  1024. }
  1025. return SUCCESS;
  1026. }
  1027. ///
  1028. /// @ingroup ge
  1029. /// @brief input zero copy node Initialize.
  1030. /// @param [in] NodePtr: Data Op.
  1031. /// @return Status
  1032. ///
  1033. Status DavinciModel::InitInputBatchLabel(const NodePtr &node) {
  1034. string batch_label;
  1035. if (!AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_BATCH_LABEL, batch_label)) {
  1036. return SUCCESS; // Not Multi-batch.
  1037. }
  1038. const auto &out_data_anchor = node->GetOutDataAnchor(kDataIndex);
  1039. GE_CHECK_NOTNULL(out_data_anchor);
  1040. for (const auto &peer_in_data_anchor : out_data_anchor->GetPeerInDataAnchors()) {
  1041. const auto &node = peer_in_data_anchor->GetOwnerNode();
  1042. const auto &op_desc = node->GetOpDesc();
  1043. GE_CHECK_NOTNULL(op_desc);
  1044. if (zero_copy_op_id_batch_label_.find(op_desc->GetId()) == zero_copy_op_id_batch_label_.end()) {
  1045. zero_copy_op_id_batch_label_[op_desc->GetId()] = batch_label;
  1046. GELOGD("Init input zero copy nodes success, op name: %s, op id: %ld, batch label: %s", op_desc->GetName().c_str(),
  1047. op_desc->GetId(), batch_label.c_str());
  1048. }
  1049. }
  1050. return SUCCESS;
  1051. }
  1052. ///
  1053. /// @ingroup ge
  1054. /// @brief output zero copy node Initialize for Case.
  1055. /// @param [in] NodePtr: netoutput Op.
  1056. /// @return Status
  1057. ///
  1058. Status DavinciModel::InitOutputBatchLabel(const NodePtr &node) {
  1059. string batch_label;
  1060. if (!AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_BATCH_LABEL, batch_label)) {
  1061. return SUCCESS; // Not Multi-batch.
  1062. }
  1063. for (const auto &in_data_anchor : node->GetAllInDataAnchors()) {
  1064. const auto &peer_out_data_anchor = in_data_anchor->GetPeerOutAnchor();
  1065. if (peer_out_data_anchor == nullptr) {
  1066. continue;
  1067. }
  1068. const auto &peer_node = peer_out_data_anchor->GetOwnerNode();
  1069. const auto &op_desc = peer_node->GetOpDesc();
  1070. GE_CHECK_NOTNULL(op_desc);
  1071. if (zero_copy_op_id_batch_label_.find(op_desc->GetId()) == zero_copy_op_id_batch_label_.end()) {
  1072. zero_copy_op_id_batch_label_[op_desc->GetId()] = batch_label;
  1073. GELOGD("Init Output zero copy nodes success, op name: %s, op id: %ld, batch label: %s",
  1074. op_desc->GetName().c_str(), op_desc->GetId(), batch_label.c_str());
  1075. }
  1076. }
  1077. return SUCCESS;
  1078. }
  1079. /// @ingroup ge
  1080. /// @brief LabelSet Op Initialize.
  1081. /// @param [in] op_desc: LabelSet Op descriptor.
  1082. /// @return Status
  1083. Status DavinciModel::InitLabelSet(const OpDescPtr &op_desc) {
  1084. uint32_t label_index = 0;
  1085. if (!AttrUtils::GetInt(op_desc, ATTR_NAME_LABEL_SWITCH_INDEX, label_index)) {
  1086. GELOGE(INTERNAL_ERROR, "InitLabelSet: %s attr [%s] not exist.", op_desc->GetName().c_str(),
  1087. ATTR_NAME_LABEL_SWITCH_INDEX.c_str());
  1088. return INTERNAL_ERROR;
  1089. }
  1090. if (label_index >= LabelNum()) {
  1091. GELOGE(INTERNAL_ERROR, "InitLabelSet: label index: %u >= label size: %zu.", label_index, LabelNum());
  1092. return INTERNAL_ERROR;
  1093. }
  1094. if (label_id_indication_.count(label_index) > 0) {
  1095. GELOGE(INTERNAL_ERROR, "InitLabelSet: %s label index: %u already used.", op_desc->GetName().c_str(), label_index);
  1096. return INTERNAL_ERROR;
  1097. }
  1098. rtStream_t stream = nullptr;
  1099. uint32_t stream_id = static_cast<uint32_t>(op_desc->GetStreamId());
  1100. if (stream_list_.size() == 1) {
  1101. stream = stream_list_[0];
  1102. } else if (stream_list_.size() > stream_id) {
  1103. stream = stream_list_[stream_id];
  1104. } else {
  1105. GELOGE(INTERNAL_ERROR, "InitLabelSet: stream index: %u >= stream size: %zu.", stream_id, stream_list_.size());
  1106. return INTERNAL_ERROR;
  1107. }
  1108. rtLabel_t rt_label = nullptr;
  1109. rtError_t rt_error = rtLabelCreateEx(&rt_label, stream);
  1110. if (rt_error != RT_ERROR_NONE || rt_label == nullptr) {
  1111. GELOGE(INTERNAL_ERROR, "InitLabelSet: %s create label failed, error=0x%x.", op_desc->GetName().c_str(), rt_error);
  1112. return INTERNAL_ERROR;
  1113. }
  1114. GELOGI("InitLabelSet: label[%u]=%p stream[%u]=%p.", label_index, rt_label, stream_id, stream);
  1115. label_id_indication_.insert(label_index);
  1116. label_list_[label_index] = rt_label;
  1117. return SUCCESS;
  1118. }
  1119. Status DavinciModel::InitVariable(const OpDescPtr &op_desc) {
  1120. variable_op_list_.push_back(op_desc);
  1121. return SUCCESS;
  1122. }
  1123. /// @ingroup ge
  1124. /// @brief ACL case, Load task list with queue.
  1125. /// @param [in] input_queue_ids: input queue ids from user, nums equal Data Op.
  1126. /// @param [in] output_queue_ids: input queue ids from user, nums equal NetOutput Op.
  1127. /// @return: 0 for success / others for failed
  1128. Status DavinciModel::SetQueIds(const std::vector<uint32_t> &input_queue_ids,
  1129. const std::vector<uint32_t> &output_queue_ids) {
  1130. if (input_queue_ids.empty() && output_queue_ids.empty()) {
  1131. GELOGE(GE_EXEC_MODEL_QUEUE_ID_INVALID, "Param is empty");
  1132. return GE_EXEC_MODEL_QUEUE_ID_INVALID;
  1133. }
  1134. input_queue_ids_ = input_queue_ids;
  1135. output_queue_ids_ = output_queue_ids;
  1136. return SUCCESS;
  1137. }
  1138. ///
  1139. /// @ingroup ge
  1140. /// @brief ACL case, Load task list with queue.
  1141. /// @param [in] input_que_ids: input queue ids from user, nums equal Data Op.
  1142. /// @param [in] output_que_ids: input queue ids from user, nums equal NetOutput Op.
  1143. /// @return: 0 for success / others for failed
  1144. ///
  1145. Status DavinciModel::LoadWithQueue() {
  1146. if (input_queue_ids_.empty() && output_queue_ids_.empty()) {
  1147. return SUCCESS;
  1148. }
  1149. if (input_queue_ids_.size() != new_input_data_info_.size()) {
  1150. GELOGE(GE_EXEC_MODEL_QUEUE_ID_INVALID, "Input queue ids not match model: input_queue=%zu input_data=%zu",
  1151. input_queue_ids_.size(), new_input_data_info_.size());
  1152. return GE_EXEC_MODEL_QUEUE_ID_INVALID;
  1153. }
  1154. if (output_queue_ids_.size() != new_output_data_info_.size()) {
  1155. GELOGE(GE_EXEC_MODEL_QUEUE_ID_INVALID, "Output queue ids not match model: output_queue=%zu output_data=%zu",
  1156. output_queue_ids_.size(), new_output_data_info_.size());
  1157. return GE_EXEC_MODEL_QUEUE_ID_INVALID;
  1158. }
  1159. GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed.");
  1160. // Binding input_queue and Data Op.
  1161. GE_CHK_STATUS_RET(BindInputQueue(), "Launch bind input queue failed.");
  1162. GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(input_mbuf_list_, new_input_outside_addrs_), "Launch zero copy failed.");
  1163. // Binding output_queue and NetOutput Op.
  1164. GE_CHK_STATUS_RET(BindOutputQueue(), "Launch bind output queue failed.");
  1165. GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(output_mbuf_list_, new_output_outside_addrs_), "Launch zero copy failed.");
  1166. GE_CHK_STATUS_RET(CpuActiveStream(), "Launch active entry stream failed.");
  1167. GE_CHK_STATUS_RET(CpuWaitEndGraph(), "Launch wait end graph failed.");
  1168. GE_CHK_STATUS_RET(BindEnqueue(), "Launch enqueue failed.");
  1169. GE_CHK_STATUS_RET(CpuModelRepeat(), "Launch model repeat failed.");
  1170. return SUCCESS;
  1171. }
  1172. /// @ingroup ge
  1173. /// @brief queue schedule, Bind input queue to Data output address.
  1174. /// @return: 0 for success / others for failed
  1175. Status DavinciModel::BindInputQueue() {
  1176. // Caller checked: input_queue_ids_.size() == input_size_list_.size() != input_addr_list_.size()
  1177. for (size_t i = 0; i < input_queue_ids_.size(); ++i) {
  1178. auto it = new_input_data_info_.find(i);
  1179. if (it == new_input_data_info_.end()) {
  1180. GELOGE(FAILED, "Input not match: tensor num=%zu, Queue id index=%zu", new_input_data_info_.size(), i);
  1181. return FAILED;
  1182. }
  1183. uint32_t queue_id = input_queue_ids_[i];
  1184. if (it->second.GetDataInfo().empty()) {
  1185. GELOGE(INTERNAL_ERROR, "the %zu input_queue not set data_info.", i);
  1186. return INTERNAL_ERROR;
  1187. }
  1188. uint32_t data_size = static_cast<uint32_t>(it->second.GetDataInfo().at(0).first);
  1189. uintptr_t data_addr = reinterpret_cast<uintptr_t>(it->second.GetDataInfo().at(0).second);
  1190. GELOGI("BindInputToQueue: graph_%u index[%zu] queue id[%u] output addr[0x%lx] output size[%u]",
  1191. runtime_param_.graph_id, i, queue_id, data_addr, data_size);
  1192. rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_INPUT_QUEUE);
  1193. if (rt_ret != RT_ERROR_NONE) {
  1194. GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret);
  1195. return RT_ERROR_TO_GE_STATUS(rt_ret);
  1196. }
  1197. if (CpuModelDequeue(queue_id) != SUCCESS) {
  1198. return INTERNAL_ERROR;
  1199. }
  1200. }
  1201. return SUCCESS;
  1202. }
  1203. /// @ingroup ge
  1204. /// @brief definiteness queue schedule, bind input queue to task.
  1205. /// @param [in] queue_id: input queue id from user.
  1206. /// @return: 0 for success / others for failed
  1207. Status DavinciModel::CpuModelDequeue(uint32_t queue_id) {
  1208. GELOGI("Set CpuKernel model dequeue task enter.");
  1209. std::shared_ptr<CpuTaskModelDequeue> dequeue_task = MakeShared<CpuTaskModelDequeue>(rt_entry_stream_);
  1210. if (dequeue_task == nullptr) {
  1211. GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelDequeue task failed.");
  1212. return MEMALLOC_FAILED;
  1213. }
  1214. // Get DataOp Output address and bind to queue.
  1215. uintptr_t in_mbuf = 0;
  1216. Status status = dequeue_task->Init(queue_id, in_mbuf);
  1217. if (status != SUCCESS) {
  1218. return status;
  1219. }
  1220. cpu_task_list_.push_back(dequeue_task);
  1221. input_mbuf_list_.push_back(in_mbuf);
  1222. GELOGI("Set CpuKernel model dequeue task success.");
  1223. return SUCCESS;
  1224. }
  1225. Status DavinciModel::CpuTaskModelZeroCopy(std::vector<uintptr_t> &mbuf_list,
  1226. std::map<const void *, ZeroCopyOffset> &outside_addrs) {
  1227. GELOGI("Set CpuKernel model zero_copy task enter.");
  1228. std::shared_ptr<CpuTaskZeroCopy> zero_copy = MakeShared<CpuTaskZeroCopy>(rt_entry_stream_);
  1229. if (zero_copy == nullptr) {
  1230. GELOGE(MEMALLOC_FAILED, "Make CpuTaskZeroCopy task failed.");
  1231. return MEMALLOC_FAILED;
  1232. }
  1233. // mdc zero_copy not support l2 fusion
  1234. Status status = zero_copy->Init(mbuf_list, outside_addrs);
  1235. if (status != SUCCESS) {
  1236. return status;
  1237. }
  1238. cpu_task_list_.push_back(zero_copy);
  1239. GELOGI("Set CpuKernel model zero_copy task success.");
  1240. return SUCCESS;
  1241. }
  1242. /// @ingroup ge
  1243. /// @brief queue schedule, bind output queue to NetOutput input address.
  1244. /// @return: 0 for success / others for failed
  1245. Status DavinciModel::BindOutputQueue() {
  1246. // Caller checked: input_queue_ids_.size() == input_size_list_.size() != input_addr_list_.size()
  1247. for (size_t i = 0; i < output_queue_ids_.size(); ++i) {
  1248. auto it = new_output_data_info_.find(i);
  1249. if (it == new_output_data_info_.end()) {
  1250. GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", new_output_data_info_.size(), i);
  1251. return FAILED;
  1252. }
  1253. uint32_t queue_id = output_queue_ids_[i];
  1254. if (it->second.GetDataInfo().empty()) {
  1255. GELOGE(INTERNAL_ERROR, "the %zu output_queue not set data_info.", i);
  1256. return INTERNAL_ERROR;
  1257. }
  1258. uint32_t data_size = static_cast<uint32_t>(it->second.GetDataInfo().at(0).first);
  1259. uintptr_t data_addr = reinterpret_cast<uintptr_t>(it->second.GetDataInfo().at(0).second);
  1260. GELOGI("BindOutputToQueue: graph_%u index[%zu] queue id[%u] input addr[0x%lx] input size[%u]",
  1261. runtime_param_.graph_id, i, queue_id, data_addr, data_size);
  1262. rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_OUTPUT_QUEUE);
  1263. if (rt_ret != RT_ERROR_NONE) {
  1264. GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret);
  1265. return RT_ERROR_TO_GE_STATUS(rt_ret);
  1266. }
  1267. Status status = CpuModelPrepareOutput(data_addr, data_size);
  1268. if (status != SUCCESS) {
  1269. return status;
  1270. }
  1271. }
  1272. return SUCCESS;
  1273. }
  1274. /// @ingroup ge
  1275. /// @brief definiteness queue schedule, bind output queue to task.
  1276. /// @param [in] addr: NetOutput Op input tensor address.
  1277. /// @param [in] size: NetOutput Op input tensor size.
  1278. /// @return: 0 for success / others for failed
  1279. Status DavinciModel::CpuModelPrepareOutput(uintptr_t addr, uint32_t size) {
  1280. GELOGI("Set CpuKernel model enqueue task enter.");
  1281. if (input_mbuf_list_.empty()) {
  1282. GELOGE(FAILED, "Need input mbuf for fill output mbuf head info.");
  1283. return FAILED;
  1284. }
  1285. std::shared_ptr<CpuTaskPrepareOutput> prepare_output = MakeShared<CpuTaskPrepareOutput>(rt_entry_stream_);
  1286. if (prepare_output == nullptr) {
  1287. GELOGE(MEMALLOC_FAILED, "Make CpuTaskPrepareOutput task failed.");
  1288. return MEMALLOC_FAILED;
  1289. }
  1290. uintptr_t out_mbuf = 0;
  1291. if (prepare_output->Init(addr, size, input_mbuf_list_.back(), out_mbuf) != SUCCESS) {
  1292. return FAILED;
  1293. }
  1294. cpu_task_list_.push_back(prepare_output);
  1295. output_mbuf_list_.push_back(out_mbuf);
  1296. GELOGI("Set CpuKernel model enqueue task success.");
  1297. return SUCCESS;
  1298. }
  1299. ///
  1300. /// @ingroup ge
  1301. /// @brief definiteness queue schedule, active original model stream.
  1302. /// @return: 0 for success / others for failed
  1303. ///
  1304. Status DavinciModel::CpuActiveStream() {
  1305. GELOGI("Set CpuKernel active stream task enter.");
  1306. std::shared_ptr<CpuTaskActiveEntry> active_entry = MakeShared<CpuTaskActiveEntry>(rt_entry_stream_);
  1307. if (active_entry == nullptr) {
  1308. GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed.");
  1309. return MEMALLOC_FAILED;
  1310. }
  1311. Status status = active_entry->Init(rt_head_stream_);
  1312. if (status != SUCCESS) {
  1313. return status;
  1314. }
  1315. cpu_task_list_.push_back(active_entry);
  1316. GELOGI("Set CpuKernel active stream task success.");
  1317. return SUCCESS;
  1318. }
  1319. /// @ingroup ge
  1320. /// @brief definiteness queue schedule, wait for end graph.
  1321. /// @return: 0 for success / others for failed
  1322. Status DavinciModel::CpuWaitEndGraph() {
  1323. GELOGI("Set CpuKernel wait end graph task enter.");
  1324. std::shared_ptr<CpuTaskWaitEndGraph> wait_endgraph = MakeShared<CpuTaskWaitEndGraph>(rt_entry_stream_);
  1325. if (wait_endgraph == nullptr) {
  1326. GELOGE(MEMALLOC_FAILED, "Make CpuTaskWaitEndGraph task failed.");
  1327. return MEMALLOC_FAILED;
  1328. }
  1329. Status status = wait_endgraph->Init(runtime_model_id_);
  1330. if (status != SUCCESS) {
  1331. return status;
  1332. }
  1333. cpu_task_list_.push_back(wait_endgraph);
  1334. GELOGI("Set CpuKernel wait end graph task success.");
  1335. return SUCCESS;
  1336. }
  1337. Status DavinciModel::BindEnqueue() {
  1338. for (size_t i = 0; i < output_queue_ids_.size(); ++i) {
  1339. auto it = new_output_data_info_.find(i);
  1340. if (it == new_output_data_info_.end()) {
  1341. GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", new_output_data_info_.size(), i);
  1342. return FAILED;
  1343. }
  1344. uint32_t queue_id = output_queue_ids_[i];
  1345. if (CpuModelEnqueue(queue_id, output_mbuf_list_[i]) != SUCCESS) {
  1346. return INTERNAL_ERROR;
  1347. }
  1348. }
  1349. return SUCCESS;
  1350. }
  1351. Status DavinciModel::CpuModelEnqueue(uint32_t queue_id, uintptr_t out_mbuf) {
  1352. GELOGI("Set CpuKernel model enqueue task enter.");
  1353. std::shared_ptr<CpuTaskModelEnqueue> model_enqueue = MakeShared<CpuTaskModelEnqueue>(rt_entry_stream_);
  1354. if (model_enqueue == nullptr) {
  1355. GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelEnqueue task failed.");
  1356. return MEMALLOC_FAILED;
  1357. }
  1358. Status status = model_enqueue->Init(queue_id, out_mbuf);
  1359. if (status != SUCCESS) {
  1360. return status;
  1361. }
  1362. cpu_task_list_.push_back(model_enqueue);
  1363. GELOGI("Set CpuKernel model enqueue task enter.");
  1364. return SUCCESS;
  1365. }
  1366. /// @ingroup ge
  1367. /// @brief definiteness queue schedule, repeat run model.
  1368. /// @return: 0 for success / others for failed
  1369. Status DavinciModel::CpuModelRepeat() {
  1370. GELOGI("Set CpuKernel repeat task enter.");
  1371. std::shared_ptr<CpuTaskModelRepeat> model_repeat = MakeShared<CpuTaskModelRepeat>(rt_entry_stream_);
  1372. if (model_repeat == nullptr) {
  1373. GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelRepeat task failed.");
  1374. return MEMALLOC_FAILED;
  1375. }
  1376. Status status = model_repeat->Init(runtime_model_id_);
  1377. if (status != SUCCESS) {
  1378. return status;
  1379. }
  1380. cpu_task_list_.push_back(model_repeat);
  1381. GELOGI("Set CpuKernel repeat task success.");
  1382. return SUCCESS;
  1383. }
  1384. Status DavinciModel::GetInputOutputDescInfo(vector<InputOutputDescInfo> &input_desc,
  1385. vector<InputOutputDescInfo> &output_desc) {
  1386. if ((data_op_list_.empty()) || (data_op_list_[0]->GetInputsSize()) != 1) {
  1387. GELOGI("data_op_list_ is empty or input_desc size is not 1.");
  1388. } else {
  1389. std::vector<uint32_t> input_formats;
  1390. GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed.");
  1391. }
  1392. std::vector<uint32_t> outputFormats;
  1393. GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, outputFormats), "get output desc info failed.");
  1394. return SUCCESS;
  1395. }
  1396. Status DavinciModel::GetInputOutputDescInfo(vector<InputOutputDescInfo> &input_desc,
  1397. vector<InputOutputDescInfo> &output_desc,
  1398. std::vector<uint32_t> &input_formats,
  1399. std::vector<uint32_t> &outputFormats) {
  1400. if ((data_op_list_.empty()) || (data_op_list_[0]->GetInputsSize()) != 1) {
  1401. GELOGE(FAILED, "OP List Pointer is null or input_desc size is not 1!");
  1402. return FAILED;
  1403. }
  1404. GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed");
  1405. GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, outputFormats), "get ouput desc info failed");
  1406. return SUCCESS;
  1407. }
  1408. ///
  1409. /// @ingroup ge
  1410. /// @brief Get dynamic batch_info
  1411. /// @param [out] batch_info
  1412. /// @param [out] dynamic_type
  1413. /// @return execute result
  1414. ///
  1415. Status DavinciModel::GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type) const {
  1416. dynamic_type = dynamic_type_;
  1417. batch_info = batch_info_;
  1418. return SUCCESS;
  1419. }
  1420. ///
  1421. /// @ingroup ge
  1422. /// @brief Get combined dynamic dims info
  1423. /// @param [out] batch_info
  1424. /// @return None
  1425. ///
  1426. void DavinciModel::GetCombinedDynamicDims(std::vector<std::vector<int64_t>> &batch_info) const {
  1427. batch_info.clear();
  1428. batch_info = combined_batch_info_;
  1429. }
  1430. ///
  1431. /// @ingroup ge
  1432. /// @brief Get user designate shape order
  1433. /// @param [out] user_input_shape_order
  1434. /// @return None
  1435. ///
  1436. void DavinciModel::GetUserDesignateShapeOrder(std::vector<std::string> &user_input_shape_order) const {
  1437. user_input_shape_order.clear();
  1438. user_input_shape_order = user_designate_shape_order_;
  1439. }
  1440. ///
  1441. /// @ingroup ge
  1442. /// @brief Get AIPP input info
  1443. /// @param [in] index
  1444. /// @param [out] aipp_info
  1445. /// @return execute result
  1446. ///
  1447. Status DavinciModel::GetAIPPInfo(uint32_t index, AippConfigInfo &aipp_info) {
  1448. GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index);
  1449. OpDescPtr data_op = data_op_list_[index];
  1450. if (!data_op->HasAttr(ATTR_NAME_AIPP)) {
  1451. GELOGW("GetAIPPInfo: there is not AIPP related with index %u.", index);
  1452. return GE_AIPP_NOT_EXIST;
  1453. }
  1454. std::unique_ptr<domi::AippOpParams> aipp_params(new (std::nothrow) domi::AippOpParams());
  1455. GE_CHECK_NOTNULL(aipp_params);
  1456. ge::GeAttrValue::NAMED_ATTRS aipp_attr;
  1457. GE_CHK_BOOL_RET_STATUS(AttrUtils::GetNamedAttrs(data_op, ATTR_NAME_AIPP, aipp_attr), GE_AIPP_NOT_EXIST,
  1458. "Data node do not contain param aipp!");
  1459. GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aipp_params.get()), "get aipp params failed");
  1460. GELOGI("GetAIPPInfo: node data: %s, type: %s, current index: %u, current node related input rank: %u",
  1461. data_op->GetName().c_str(), data_op->GetType().c_str(), index, aipp_params->related_input_rank());
  1462. GE_CHK_STATUS_RET(AippUtils::ConvertAippParams2AippInfo(aipp_params.get(), aipp_info),
  1463. "convert aipp params to aipp config info failed");
  1464. return SUCCESS;
  1465. }
  1466. Status DavinciModel::GetAippType(uint32_t index, InputAippType &type, size_t &aipp_index) {
  1467. GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index);
  1468. // Set default value
  1469. type = DATA_WITHOUT_AIPP;
  1470. aipp_index = 0xFFFFFFFF; // default invalid value
  1471. OpDescPtr data_op = data_op_list_[index];
  1472. GE_CHECK_NOTNULL(data_op);
  1473. if (!data_op->HasAttr(ATTR_DATA_RELATED_AIPP_MODE)) {
  1474. GELOGW("There is no aipp releated info with index %u.", index);
  1475. return SUCCESS;
  1476. }
  1477. std::string data_mode;
  1478. (void)AttrUtils::GetStr(data_op, ATTR_DATA_RELATED_AIPP_MODE, data_mode);
  1479. if (data_mode == "static_aipp") {
  1480. type = DATA_WITH_STATIC_AIPP;
  1481. } else if (data_mode == "dynamic_aipp") {
  1482. type = DATA_WITH_DYNAMIC_AIPP;
  1483. } else if (data_mode == "dynamic_aipp_conf") {
  1484. type = DYNAMIC_AIPP_NODE;
  1485. } else {
  1486. GELOGE(INTERNAL_ERROR, "The info of aipp releated info %s is invalid with index %u.", data_mode.c_str(), index);
  1487. return INTERNAL_ERROR;
  1488. }
  1489. if (type == DATA_WITH_DYNAMIC_AIPP) {
  1490. string releated_name;
  1491. (void)AttrUtils::GetStr(data_op, ATTR_DATA_AIPP_DATA_NAME_MAP, releated_name);
  1492. for (size_t i = 0; i < data_op_list_.size(); ++i) {
  1493. GE_CHECK_NOTNULL(data_op_list_[i]);
  1494. if (data_op_list_[i]->GetName() == releated_name) {
  1495. GELOGI("Find aipp_data [%s] index %zu from index %u", releated_name.c_str(), i, index);
  1496. aipp_index = i;
  1497. }
  1498. }
  1499. if (aipp_index == 0xFFFFFFFF) {
  1500. GELOGE(INTERNAL_ERROR, "Can not find aipp data node from index %u", index);
  1501. return INTERNAL_ERROR;
  1502. }
  1503. }
  1504. return SUCCESS;
  1505. }
  1506. void DavinciModel::SetDynamicSize(const std::vector<uint64_t> &batch_num, int32_t dynamic_type) {
  1507. batch_size_.clear();
  1508. if (batch_num.empty()) {
  1509. GELOGD("User has not set dynammic data");
  1510. }
  1511. for (size_t i = 0; i < batch_num.size(); i++) {
  1512. batch_size_.emplace_back(batch_num[i]);
  1513. }
  1514. dynamic_type_ = dynamic_type;
  1515. }
  1516. void DavinciModel::GetCurShape(std::vector<int64_t> &batch_info, int32_t &dynamic_type) {
  1517. if (batch_size_.empty()) {
  1518. GELOGD("User does not set dynamic size");
  1519. }
  1520. for (size_t i = 0; i < batch_size_.size(); i++) {
  1521. GELOGI("Start to get current shape");
  1522. batch_info.emplace_back(batch_size_[i]);
  1523. }
  1524. dynamic_type = dynamic_type_;
  1525. }
  1526. void DavinciModel::GetModelAttr(std::vector<std::string> &dynamic_output_shape_info) {
  1527. for (auto &op : output_op_list_) {
  1528. if (op->GetType() != NETOUTPUT) {
  1529. continue;
  1530. }
  1531. GELOGI("Start to get dynamic output dims attr");
  1532. if (!AttrUtils::GetListStr(op, ATTR_NAME_DYNAMIC_OUTPUT_DIMS, dynamic_output_shape_info)) {
  1533. GELOGD("Can not get dynamic output dims attr");
  1534. }
  1535. }
  1536. }
  1537. Status DavinciModel::GetInputOutputDescInfoForZeroCopy(vector<InputOutputDescInfo> &input_desc,
  1538. vector<InputOutputDescInfo> &output_desc,
  1539. std::vector<uint32_t> &input_formats,
  1540. std::vector<uint32_t> &outputFormats) {
  1541. if ((data_op_list_.empty()) || (1 != data_op_list_[0]->GetInputsSize())) {
  1542. GELOGE(FAILED, "OP List Pointer is null or input_desc size is not 1!");
  1543. return FAILED;
  1544. }
  1545. GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats), "get input desc info failed");
  1546. GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, outputFormats), "get ouput desc info failed");
  1547. GE_CHK_BOOL_RET_STATUS(output_desc.size() == output_memory_size_list_.size(), INTERNAL_ERROR,
  1548. "output_desc size[%zu] not equal output_size_list_[%zu] size!", output_desc.size(),
  1549. output_memory_size_list_.size());
  1550. /// For function zero copy,the momery should be aligned by 512 bytes.
  1551. /// And, because of the cce op limit, size should be lager than the real shape size. The memory should be padded by 32
  1552. /// bytes.
  1553. /// *size equals to ((tensorDesc->dataSize + 2 * 32 - 1) / 32) * 32;
  1554. for (size_t i = 0; i < output_memory_size_list_.size(); i++) {
  1555. output_desc[i].size = output_memory_size_list_[i];
  1556. }
  1557. return SUCCESS;
  1558. }
  1559. void DavinciModel::SetInputDimsInfo(const vector<int64_t> &model_input_dims, Format &format,
  1560. InputOutputDescInfo &input) {
  1561. uint32_t n, c, h, w;
  1562. n = format == FORMAT_NHWC ? NHWC_DIM_N : NCHW_DIM_N;
  1563. c = format == FORMAT_NHWC ? NHWC_DIM_C : NCHW_DIM_C;
  1564. h = format == FORMAT_NHWC ? NHWC_DIM_H : NCHW_DIM_H;
  1565. w = format == FORMAT_NHWC ? NHWC_DIM_W : NCHW_DIM_W;
  1566. if (model_input_dims.size() == static_cast<size_t>(NORMAL_TENSOR_SIZE)) {
  1567. input.shape_info.num = model_input_dims[n];
  1568. input.shape_info.height = model_input_dims[h];
  1569. input.shape_info.width = model_input_dims[w];
  1570. input.shape_info.channel = model_input_dims[c];
  1571. }
  1572. for (size_t k = 0; k < model_input_dims.size(); ++k) {
  1573. input.shape_info.dims.push_back(model_input_dims[k]);
  1574. }
  1575. return;
  1576. }
  1577. void DavinciModel::CreateInputDimsInfo(const OpDescPtr &op_desc, Format format, InputOutputDescInfo &input) {
  1578. if (is_new_model_desc_ && op_desc->HasAttr(ATTR_NAME_INPUT_DIMS)) {
  1579. // When static aipp is set, need to get the model input dims which processed by aipp
  1580. vector<int64_t> model_input_dims;
  1581. (void)AttrUtils::GetListInt(op_desc, ATTR_NAME_INPUT_DIMS, model_input_dims);
  1582. SetInputDimsInfo(model_input_dims, format, input);
  1583. return;
  1584. }
  1585. // judge if this data is linked dynamic aipp first, multiply batch has been considered
  1586. if (op_desc->HasAttr(ATTR_DYNAMIC_AIPP_INPUT_DIMS)) {
  1587. vector<int64_t> dynamic_aipp_input_dims;
  1588. (void)AttrUtils::GetListInt(op_desc, ATTR_DYNAMIC_AIPP_INPUT_DIMS, dynamic_aipp_input_dims);
  1589. SetInputDimsInfo(dynamic_aipp_input_dims, format, input);
  1590. return;
  1591. } else {
  1592. // judge if this data is multiply batch
  1593. if (!op_desc->HasAttr(ATTR_MBATCH_ORIGIN_INPUT_DIMS)) {
  1594. vector<int64_t> input_dims = op_desc->GetInputDescPtr(0)->GetShape().GetDims();
  1595. SetInputDimsInfo(input_dims, format, input);
  1596. return;
  1597. } else {
  1598. vector<int64_t> origin_input_dims;
  1599. (void)AttrUtils::GetListInt(op_desc, ATTR_MBATCH_ORIGIN_INPUT_DIMS, origin_input_dims);
  1600. SetInputDimsInfo(origin_input_dims, format, input);
  1601. return;
  1602. }
  1603. }
  1604. }
  1605. Status DavinciModel::GetInputDescInfo(vector<InputOutputDescInfo> &input_desc, std::vector<uint32_t> &formats) {
  1606. for (size_t index = 0; index < data_op_list_.size(); ++index) {
  1607. InputOutputDescInfo input;
  1608. GE_CHECK_NOTNULL(data_op_list_[index]);
  1609. GE_CHECK_NOTNULL(data_op_list_[index]->GetInputDescPtr(0));
  1610. Format format = data_op_list_[index]->GetInputDescPtr(0)->GetFormat();
  1611. CreateInputDimsInfo(data_op_list_[index], format, input);
  1612. input.data_type = data_op_list_[index]->GetInputDescPtr(0)->GetDataType();
  1613. input.name = data_op_list_[index]->GetName();
  1614. int64_t input_size = 0;
  1615. GE_CHK_STATUS_RET(TensorUtils::GetSize(*data_op_list_[index]->GetInputDescPtr(0), input_size),
  1616. "get input size failed.");
  1617. input.size = input_size;
  1618. formats.push_back(format);
  1619. input_desc.push_back(input);
  1620. }
  1621. // cause GetInputDescInfo called not only once, set is_new_model_desc_ to false after calc the model input dims
  1622. is_new_model_desc_ = false;
  1623. return SUCCESS;
  1624. }
  1625. void DavinciModel::CreateOutput(uint32_t index, OpDescPtr &op_desc, InputOutputDescInfo &output,
  1626. uint32_t &format_result) {
  1627. /// netoutput input tensor desc
  1628. GE_IF_BOOL_EXEC(op_desc->GetInputDescPtr(index) == nullptr, GELOGE(FAILED, "OpDesc GetInputDescPtr is nullptr");
  1629. return );
  1630. Format format = op_desc->GetInputDescPtr(index)->GetFormat();
  1631. GeShape shape = op_desc->GetInputDescPtr(index)->GetShape();
  1632. DataType data_type = op_desc->GetInputDescPtr(index)->GetDataType();
  1633. int64_t dims[] = {1, 1, 1, 1};
  1634. format_result = format;
  1635. if (format == FORMAT_ND) { // for ND tensor
  1636. for (size_t i = 0; i < shape.GetDimNum() && i < (sizeof(dims) / sizeof(dims[0])); i++) {
  1637. dims[i] = shape.GetDim(i);
  1638. }
  1639. } else { // FOR FORMAT_NHWC or FORMAT_NCHW
  1640. dims[0] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_N : NCHW_DIM_N); // 0: first dim
  1641. dims[1] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_C : NCHW_DIM_C); // 1: second dim
  1642. dims[2] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_H : NCHW_DIM_H); // 2: third dim
  1643. dims[3] = shape.GetDim(format == FORMAT_NHWC ? NHWC_DIM_W : NCHW_DIM_W); // 3: forth dim
  1644. }
  1645. output.shape_info.num = dims[0]; // 0: first dim
  1646. output.shape_info.channel = dims[1]; // 1: second dim
  1647. output.shape_info.height = dims[2]; // 2: third dim
  1648. output.shape_info.width = dims[3]; // 3: forth dim
  1649. if (op_desc->GetInputDescPtr(index)->GetFormat() == FORMAT_FRACTAL_Z) { // FraczToHWCK
  1650. int64_t k = shape.GetDim(0); // 0: first dim
  1651. int64_t c = shape.GetDim(1); // 1: second dim
  1652. int64_t h = shape.GetDim(2); // 2: third dim
  1653. int64_t w = shape.GetDim(3); // 3: forth dim
  1654. output.shape_info.dims.push_back(h);
  1655. output.shape_info.dims.push_back(w);
  1656. output.shape_info.dims.push_back(c);
  1657. output.shape_info.dims.push_back(k);
  1658. format_result = FORMAT_HWCN;
  1659. } else {
  1660. for (size_t j = 0; j < shape.GetDimNum(); j++) {
  1661. output.shape_info.dims.push_back(shape.GetDim(j));
  1662. }
  1663. }
  1664. int64_t tensor_size = 0;
  1665. (void)TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size); // no need to check value
  1666. output.size = static_cast<uint64_t>(tensor_size);
  1667. output.data_type = op_desc->GetInputDescPtr(index)->GetDataType();
  1668. }
  1669. Status DavinciModel::GetOutputDescInfo(vector<InputOutputDescInfo> &output_desc, std::vector<uint32_t> &formats) {
  1670. GELOGI("Output node size: %zu", output_op_list_.size());
  1671. for (size_t i = 0; i < output_op_list_.size(); i++) {
  1672. auto &op_desc = output_op_list_[i];
  1673. uint32_t out_size = static_cast<uint32_t>(op_desc->GetInputsSize());
  1674. for (uint32_t index = 0; index < out_size; index++) {
  1675. string output_name;
  1676. InputOutputDescInfo output;
  1677. uint32_t format_result;
  1678. CreateOutput(index, op_desc, output, format_result);
  1679. std::vector<std::string> src_name = op_desc->GetSrcName();
  1680. std::vector<int64_t> src_index = op_desc->GetSrcIndex();
  1681. GE_CHK_BOOL_RET_STATUS(src_name.size() > index && src_index.size() > index, INTERNAL_ERROR,
  1682. "construct output_name failed.");
  1683. // forward compatbility, if old om has no out_node_name, need to return output follow origin way
  1684. if (out_size == out_node_name_.size()) {
  1685. // neweast plan, the index will add to name during generate model.
  1686. bool contains_colon = out_node_name_[index].find(":") != std::string::npos;
  1687. output_name =
  1688. contains_colon ? out_node_name_[index] : out_node_name_[index] + ":" + std::to_string(src_index[index]);
  1689. } else {
  1690. output_name = std::string("output_") + std::to_string(index) + "_" + src_name[index] + "_" +
  1691. std::to_string(src_index[index]);
  1692. }
  1693. output.name = output_name;
  1694. output_desc.push_back(output);
  1695. formats.push_back(format_result);
  1696. }
  1697. }
  1698. return SUCCESS;
  1699. }
  1700. ge::Format DavinciModel::GetFormat() {
  1701. if ((data_op_list_.empty()) || data_op_list_[0] == nullptr || data_op_list_[0]->GetInputDescPtr(0) == nullptr) {
  1702. GELOGW("OP List Pointer is null or input_desc size is not 1!");
  1703. return FORMAT_NCHW;
  1704. }
  1705. return data_op_list_[0]->GetInputDescPtr(0)->GetFormat();
  1706. }
  1707. Status DavinciModel::CopyInputData(const InputData &input_data, bool device_data) {
  1708. rtMemcpyKind_t kind = device_data ? RT_MEMCPY_DEVICE_TO_DEVICE : RT_MEMCPY_HOST_TO_DEVICE;
  1709. const std::vector<DataBuffer> &blobs = input_data.blobs;
  1710. for (const auto &data : new_input_data_info_) {
  1711. if (data.first >= blobs.size()) {
  1712. GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld", blobs.size(),
  1713. new_input_data_info_.size(), data.first, data.second.GetDataInfo().at(0).first);
  1714. return FAILED;
  1715. }
  1716. const DataBuffer &data_buf = blobs[data.first];
  1717. if (data_buf.length == 0) {
  1718. GELOGW("No data need to memcpy!");
  1719. return SUCCESS;
  1720. }
  1721. uint64_t data_size = data.second.GetDataSize();
  1722. GE_CHK_BOOL_RET_STATUS(data_size >= data_buf.length, PARAM_INVALID,
  1723. "input data size(%lu) does not match model required size(%lu), ret failed.", data_buf.length,
  1724. data_size);
  1725. void *mem_addr = data.second.GetBasicAddr();
  1726. void *data_buf_addr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(data_buf.data));
  1727. uint64_t data_buf_length = data_buf.length;
  1728. GELOGI("[IMAS]CopyPlainData memcpy graph_%lu type[F] input[%lu] dst[%p] src[%p] mem_size[%lu] datasize[%lu]",
  1729. runtime_param_.graph_id, data.first, mem_addr, data_buf_addr, data_size, data_buf_length);
  1730. GE_CHK_RT_RET(rtMemcpy(mem_addr, data_size, data_buf_addr, data_buf_length, kind));
  1731. }
  1732. return SUCCESS;
  1733. }
  1734. Status DavinciModel::SyncVarData() {
  1735. GELOGI("Sync var data, model id:%u", model_id_);
  1736. Status ret = SUCCESS;
  1737. OpDescPtr global_step = GetVariableOp(NODE_NAME_GLOBAL_STEP);
  1738. if (global_step != nullptr) {
  1739. auto v_output_size = ModelUtils::GetOutputSize(global_step);
  1740. auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param_, global_step);
  1741. if (v_output_size.empty() || v_output_addr.empty()) {
  1742. GELOGE(PARAM_INVALID, "global step op:%s not set output", global_step->GetName().c_str());
  1743. return PARAM_INVALID;
  1744. }
  1745. std::vector<uint64_t> v_step;
  1746. v_step.push_back(iterator_count_);
  1747. GE_CHK_RT_RET(rtMemcpy(v_output_addr[0], v_output_size[0], v_step.data(), v_step.size() * sizeof(uint64_t),
  1748. RT_MEMCPY_HOST_TO_DEVICE));
  1749. }
  1750. for (auto op_desc : variable_op_list_) {
  1751. ret =
  1752. VarManager::Instance(session_id_)->SyncVarData(runtime_param_.graph_id, op_desc->GetName(), op_desc, mem_base_);
  1753. GE_CHK_BOOL_EXEC(ret == SUCCESS, break, "sync var data ret failed, model id:%u, op name:%s.", model_id_,
  1754. op_desc->GetName().c_str());
  1755. }
  1756. return ret;
  1757. }
  1758. inline int64_t SumSize(const vector<int64_t> &size_list) {
  1759. int64_t sum_size = 0;
  1760. for (const int64_t &size : size_list) {
  1761. sum_size += size;
  1762. }
  1763. return sum_size;
  1764. }
  1765. Status DavinciModel::SinkModelProfile() {
  1766. // profiling plugin must be registered
  1767. Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter();
  1768. GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS);
  1769. GELOGI("Start collect model load profiling data.");
  1770. Msprof::Engine::ReporterData reporter_data{};
  1771. // report model data tag name
  1772. std::string tag_name;
  1773. tag_name.append("model_load_info_").append(std::to_string(this->Id()));
  1774. GE_CHK_BOOL_EXEC(memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN, tag_name.c_str(), tag_name.size()) == EOK,
  1775. return FAILED, "Sink model tag memcpy error.");
  1776. // Model Header
  1777. string name;
  1778. if (!om_name_.empty()) {
  1779. name = om_name_;
  1780. } else {
  1781. name = name_;
  1782. }
  1783. size_t name_len = name.size();
  1784. reporter_data.deviceId = device_id_;
  1785. reporter_data.data = (unsigned char *)&name_len;
  1786. reporter_data.dataLen = sizeof(int32_t);
  1787. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1788. this->Id());
  1789. reporter_data.data = (unsigned char *)name.c_str();
  1790. reporter_data.dataLen = name.size();
  1791. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1792. this->Id());
  1793. uint32_t model_id = this->Id();
  1794. reporter_data.data = (unsigned char *)&model_id;
  1795. reporter_data.dataLen = sizeof(uint32_t);
  1796. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1797. this->Id());
  1798. // Load Start/End Time
  1799. int64_t start_time = this->GetLoadBeginTime();
  1800. reporter_data.data = (unsigned char *)&start_time;
  1801. reporter_data.dataLen = sizeof(int64_t);
  1802. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1803. this->Id());
  1804. int64_t end_time = this->GetLoadEndTime();
  1805. reporter_data.data = (unsigned char *)&end_time;
  1806. reporter_data.dataLen = sizeof(int64_t);
  1807. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1808. this->Id());
  1809. int32_t task_num = task_list_.size();
  1810. std::multimap<uint32_t, uint32_t> op_id_map;
  1811. std::set<uint32_t> task_id_set;
  1812. for (int32_t i = 0; i < task_num; i++) {
  1813. auto task = task_list_[i];
  1814. auto fusion_op_info = task->GetFusionOpInfo();
  1815. // when type is RT_MODEL_TASK_KERNEL, ctx is not null
  1816. if (fusion_op_info != nullptr) {
  1817. uint32_t op_num = fusion_op_info->original_op_names.size();
  1818. uint32_t task_id = task->GetTaskID();
  1819. if (op_num > 0) {
  1820. GELOGI("task.id = %u, opNum = %u", task_id, op_num);
  1821. op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id));
  1822. }
  1823. }
  1824. }
  1825. struct memoryInfo {
  1826. int64_t input_size;
  1827. int64_t output_size;
  1828. int64_t weight_size;
  1829. int64_t workspace_size;
  1830. int64_t total_size;
  1831. memoryInfo() : input_size(0), output_size(0), weight_size(0), workspace_size(0), total_size(0) {}
  1832. };
  1833. using CIT = std::multimap<uint32_t, uint32_t>::const_iterator;
  1834. using Range = std::pair<CIT, CIT>;
  1835. for (int32_t i = 0; i < task_num; i++) {
  1836. auto task = task_list_[i];
  1837. auto fusion_op_info = task->GetFusionOpInfo();
  1838. if (fusion_op_info != nullptr && fusion_op_info->original_op_names.size() > 0) {
  1839. uint32_t task_id = task->GetTaskID();
  1840. uint32_t op_num = fusion_op_info->original_op_names.size();
  1841. uint32_t task_count = 0;
  1842. if (task_id_set.count(task_id) != 0) {
  1843. continue;
  1844. }
  1845. uint32_t op_id = fusion_op_info->op_index;
  1846. Range range = op_id_map.equal_range(op_id);
  1847. for (CIT range_idx = range.first; range_idx != range.second; ++range_idx) {
  1848. task_count++;
  1849. uint32_t task_id = range_idx->second;
  1850. task_id_set.insert(task_id);
  1851. }
  1852. // op name after fusion
  1853. string fusion_op_name = fusion_op_info->op_name;
  1854. int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size();
  1855. reporter_data.data = (unsigned char *)&fusion_op_name_len;
  1856. reporter_data.dataLen = sizeof(int32_t);
  1857. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1858. this->Id());
  1859. reporter_data.data = (unsigned char *)fusion_op_name.c_str();
  1860. reporter_data.dataLen = fusion_op_name_len;
  1861. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1862. this->Id());
  1863. // original op name before fusion
  1864. reporter_data.data = (unsigned char *)&op_num;
  1865. reporter_data.dataLen = sizeof(int32_t);
  1866. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1867. this->Id());
  1868. for (uint32_t k = 0; k < op_num; k++) {
  1869. std::string op_name = fusion_op_info->original_op_names[k];
  1870. int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size();
  1871. reporter_data.data = (unsigned char *)&op_name_len;
  1872. reporter_data.dataLen = sizeof(int32_t);
  1873. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1874. this->Id());
  1875. reporter_data.data = (unsigned char *)op_name.c_str();
  1876. reporter_data.dataLen = op_name_len;
  1877. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1878. this->Id());
  1879. }
  1880. // stream id info
  1881. uint32_t streamId = task->GetStreamId();
  1882. reporter_data.data = (unsigned char *)&streamId;
  1883. reporter_data.dataLen = sizeof(int32_t);
  1884. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1885. this->Id());
  1886. // memory info
  1887. struct memoryInfo memory_info;
  1888. uint32_t op_index = fusion_op_info->op_index;
  1889. auto iter = op_list_.find(op_index);
  1890. GE_CHK_BOOL_EXEC(iter != op_list_.end(), return FAILED, "index is out of range, index: %u", op_index);
  1891. auto op_desc = iter->second;
  1892. memory_info.input_size = SumSize(ModelUtils::GetInputSize(op_desc));
  1893. memory_info.output_size = SumSize(ModelUtils::GetOutputSize(op_desc));
  1894. memory_info.workspace_size = SumSize(ModelUtils::GetWorkspaceSize(op_desc));
  1895. memory_info.weight_size = SumSize(ModelUtils::GetWeightSize(op_desc));
  1896. memory_info.total_size =
  1897. memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size;
  1898. reporter_data.data = (unsigned char *)&memory_info;
  1899. reporter_data.dataLen = sizeof(struct memoryInfo);
  1900. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1901. this->Id());
  1902. // task info
  1903. reporter_data.data = (unsigned char *)&task_count;
  1904. reporter_data.dataLen = sizeof(uint32_t);
  1905. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1906. this->Id());
  1907. Range task_range = op_id_map.equal_range(op_id);
  1908. for (CIT idx = task_range.first; idx != task_range.second; ++idx) {
  1909. uint32_t task_id = idx->second;
  1910. reporter_data.data = (unsigned char *)&task_id;
  1911. reporter_data.dataLen = sizeof(uint32_t);
  1912. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1913. this->Id());
  1914. }
  1915. }
  1916. }
  1917. return SUCCESS;
  1918. }
  1919. Status DavinciModel::SinkTimeProfile(const InputData &current_data) {
  1920. // profiling plugin must be registered
  1921. Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter();
  1922. GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS);
  1923. Msprof::Engine::ReporterData reporter_data{};
  1924. // report model data tag name
  1925. std::string tag_name;
  1926. tag_name.append("model_time_info_")
  1927. .append(std::to_string(this->Id()))
  1928. .append("_")
  1929. .append(std::to_string(current_data.index));
  1930. GE_CHK_BOOL_EXEC(memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN, tag_name.c_str(), tag_name.size()) == EOK,
  1931. return FAILED, "Sink model tag memcpy error.");
  1932. // device id
  1933. reporter_data.deviceId = device_id_;
  1934. // Model Header
  1935. string name;
  1936. if (!om_name_.empty()) {
  1937. name = om_name_;
  1938. } else {
  1939. name = name_;
  1940. }
  1941. size_t name_len = name.size();
  1942. reporter_data.data = (unsigned char *)&name_len;
  1943. reporter_data.dataLen = sizeof(int32_t);
  1944. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1945. this->Id());
  1946. reporter_data.data = (unsigned char *)name.c_str();
  1947. reporter_data.dataLen = name.size();
  1948. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.",
  1949. this->Id());
  1950. // request id
  1951. uint64_t request_id = current_data.request_id;
  1952. reporter_data.data = (unsigned char *)&request_id;
  1953. reporter_data.dataLen = sizeof(uint32_t);
  1954. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED,
  1955. "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index);
  1956. // thread id
  1957. int32_t thread_id = GetDataInputTid();
  1958. reporter_data.data = (unsigned char *)&thread_id;
  1959. reporter_data.dataLen = sizeof(int32_t);
  1960. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED,
  1961. "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index);
  1962. // time info
  1963. time_info_.modelId = this->Id();
  1964. reporter_data.data = (unsigned char *)&time_info_;
  1965. reporter_data.dataLen = sizeof(struct timeInfo);
  1966. GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED,
  1967. "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index);
  1968. return SUCCESS;
  1969. }
  1970. void DavinciModel::SetProfileTime(ModelProcStage stage, int64_t endTime) {
  1971. int64_t time = endTime;
  1972. if (time == 0) {
  1973. mmTimespec timespec = mmGetTickCount();
  1974. time = timespec.tv_sec * 1000 * 1000 * 1000 + timespec.tv_nsec; // 1000 ^ 3 converts second to nanosecond
  1975. }
  1976. switch (stage) {
  1977. case MODEL_LOAD_START:
  1978. load_begin_time_ = time;
  1979. break;
  1980. case MODEL_LOAD_END:
  1981. load_end_time_ = time;
  1982. break;
  1983. case MODEL_PRE_PROC_START:
  1984. time_info_.processBeginTime = time;
  1985. break;
  1986. case MODEL_PRE_PROC_END:
  1987. time_info_.processEndTime = time;
  1988. break;
  1989. case MODEL_INFER_START:
  1990. time_info_.inferenceBeginTime = time;
  1991. break;
  1992. case MODEL_INFER_END:
  1993. time_info_.inferenceEndTime = time;
  1994. break;
  1995. case MODEL_AFTER_PROC_START:
  1996. time_info_.dumpBeginTime = time;
  1997. break;
  1998. case MODEL_AFTER_PROC_END:
  1999. time_info_.dumpEndTime = time;
  2000. break;
  2001. default:
  2002. break;
  2003. }
  2004. return;
  2005. }
  2006. ///
  2007. /// @ingroup ge
  2008. /// @brief send Output Op result to upper layer
  2009. /// @already malloced in ModelLoad, no need to malloc again
  2010. /// @param [in] data_id: the index of output_data
  2011. /// @param [in/out] output_data: real user output_data
  2012. /// @param [in] kind: the kind of rtMemcpy
  2013. /// @return Status result
  2014. /// @author
  2015. ///
  2016. Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, rtMemcpyKind_t kind) {
  2017. if (output_op_list_.empty()) {
  2018. Status ret = SyncVarData();
  2019. return ret;
  2020. }
  2021. output_data.index = data_id;
  2022. output_data.model_id = model_id_;
  2023. if (output_data.blobs.size() != new_output_data_info_.size()) {
  2024. GELOGE(FAILED, "Output data buffer num=%zu not equal model data num=%zu", output_data.blobs.size(),
  2025. new_output_data_info_.size());
  2026. return FAILED;
  2027. }
  2028. std::vector<DataBuffer> &blobs = output_data.blobs;
  2029. for (const auto &output : new_output_data_info_) {
  2030. if (output.first >= blobs.size()) {
  2031. GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld", blobs.size(),
  2032. new_input_data_info_.size(), output.first, output.second.GetDataInfo().at(0).first);
  2033. return FAILED;
  2034. }
  2035. if ((kind == RT_MEMCPY_DEVICE_TO_DEVICE) && (copy_only_addrs_.count(output.second.GetBasicAddr()) == 0)) {
  2036. continue; // Skip: Feed by zero copy.
  2037. }
  2038. DataBuffer &buffer = blobs[output.first];
  2039. uint64_t mem_size = static_cast<uint64_t>(output.second.GetDataSize());
  2040. if ((buffer.length == 0) || (mem_size == 0)) {
  2041. GELOGI("Length of data is zero, No need copy. output tensor index=%u", output.first);
  2042. continue;
  2043. }
  2044. if (is_dynamic_) {
  2045. GELOGI("No need to check output data size.");
  2046. } else if (buffer.length < mem_size) {
  2047. GELOGE(FAILED, "Tensor data size=%lu, buffer size=%u", mem_size, buffer.length);
  2048. return FAILED;
  2049. } else if (buffer.length > mem_size) {
  2050. GELOGW("Tensor data size=%lu, buffer size=%u", mem_size, buffer.length);
  2051. }
  2052. uint64_t data_size = output.second.GetDataSize();
  2053. uint64_t buffer_length = buffer.length;
  2054. void *buffer_addr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(buffer.data));
  2055. GELOGI("[IMAS]CopyPlainData memcpy graph_%u type[F] output[%u] memaddr[%p] mem_size[%lu] datasize[%u]",
  2056. runtime_param_.graph_id, output.first, output.second.GetBasicAddr(), data_size, buffer_length);
  2057. GE_CHK_RT_RET(rtMemcpy(buffer_addr, buffer_length, output.second.GetBasicAddr(), data_size, kind));
  2058. }
  2059. return SUCCESS;
  2060. }
  2061. Status DavinciModel::GenOutputTensorInfo(const OpDescPtr &op_desc, uint32_t data_index, OutputData *output_data,
  2062. std::vector<ge::OutputTensorInfo> &outputs) {
  2063. GE_CHECK_NOTNULL(op_desc);
  2064. GE_CHECK_NOTNULL(output_data);
  2065. if (output_data->blobs.size() > data_index) {
  2066. GELOGI("No need to generate output tensor info, model id:%u", model_id_);
  2067. return SUCCESS;
  2068. }
  2069. std::vector<int64_t> out_buffer_size_vec;
  2070. std::vector<std::vector<int64_t>> shape_info_vec;
  2071. size_t input_num = op_desc->GetInputsSize();
  2072. for (size_t i = 0; i < input_num; ++i) {
  2073. int64_t size = 0;
  2074. auto input_desc = op_desc->GetInputDescPtr(i);
  2075. GE_CHECK_NOTNULL(input_desc);
  2076. auto ret = TensorUtils::GetTensorSizeInBytes(*input_desc, size);
  2077. if (ret != GRAPH_SUCCESS) {
  2078. GELOGE(ret, "Get size from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i);
  2079. return ret;
  2080. }
  2081. out_buffer_size_vec.push_back(size);
  2082. shape_info_vec.push_back(input_desc->GetShape().GetDims());
  2083. }
  2084. GELOGI("Output blobs size:%zu, data index:%u, model id:%u", out_buffer_size_vec.size(), data_index, model_id_);
  2085. for (size_t i = 0; i < out_buffer_size_vec.size(); ++i) {
  2086. std::unique_ptr<uint8_t[]> data_buf(new (std::nothrow) uint8_t[out_buffer_size_vec[i]]);
  2087. if (data_buf == nullptr) {
  2088. GELOGE(GE_GRAPH_MALLOC_FAILED, "Malloc buffer failed.");
  2089. return GE_GRAPH_MALLOC_FAILED;
  2090. }
  2091. output_data->blobs.push_back({data_buf.get(), static_cast<uint64_t>(out_buffer_size_vec[i]), false});
  2092. ge::OutputTensorInfo output;
  2093. output.dims = shape_info_vec[i];
  2094. output.data = std::move(data_buf);
  2095. output.length = out_buffer_size_vec[i];
  2096. outputs.emplace_back(std::move(output));
  2097. GELOGI("Output index:%zu, data_length:%lu.", i, output.length);
  2098. }
  2099. return SUCCESS;
  2100. }
  2101. ///
  2102. /// @ingroup ge
  2103. /// @brief send Output Op result to upper layer
  2104. /// @already malloced in ModelLoad, no need to malloc again
  2105. /// @param [in] data_id: the index of output_data
  2106. /// @param [in] rslt_flg: result flag
  2107. /// @param [in] seq_end_flag: sequence end flag
  2108. /// @param [out] output_data: real user output_data
  2109. /// @return Status result
  2110. /// @author
  2111. ///
  2112. Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const bool seq_end_flag,
  2113. OutputData *output_data) {
  2114. GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null.");
  2115. std::vector<ge::OutputTensorInfo> outputs;
  2116. // return result is not required
  2117. if (!rslt_flg && !seq_end_flag) {
  2118. GELOGW("Compute failed, model id: %u", model_id_);
  2119. auto model_manager = ModelManager::GetInstance();
  2120. GE_CHECK_NOTNULL(model_manager);
  2121. auto exception_infos = model_manager->GetExceptionInfos();
  2122. if (exception_infos.size() > 0) {
  2123. GE_CHK_STATUS_RET(data_dumper_.DumpExceptionInfo(exception_infos), "Dump exception info failed");
  2124. } else {
  2125. GELOGI("Exception info is null");
  2126. }
  2127. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed.");
  2128. return INTERNAL_ERROR;
  2129. }
  2130. if (output_op_list_.empty()) {
  2131. GELOGW("Output tensor list is empty, model id: %u", model_id_);
  2132. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed.");
  2133. return INTERNAL_ERROR;
  2134. }
  2135. GE_CHECK_NOTNULL(output_data);
  2136. // index of data in output_data
  2137. uint32_t data_index = 0;
  2138. output_data->index = data_id;
  2139. output_data->model_id = model_id_;
  2140. // copy output data from op to designated position
  2141. for (auto &op_desc : output_op_list_) {
  2142. if (GenOutputTensorInfo(op_desc, data_index, output_data, outputs) != SUCCESS) {
  2143. return INTERNAL_ERROR;
  2144. }
  2145. data_index += op_desc->GetInputsSize();
  2146. }
  2147. if (CopyOutputData(data_id, *output_data, RT_MEMCPY_DEVICE_TO_HOST) != SUCCESS) {
  2148. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed");
  2149. return INTERNAL_ERROR;
  2150. }
  2151. if (seq_end_flag) {
  2152. GELOGW("End of sequence, model id: %u", model_id_);
  2153. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, END_OF_SEQUENCE, outputs), "OnCompute Done failed.");
  2154. return END_OF_SEQUENCE;
  2155. }
  2156. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed");
  2157. return SUCCESS;
  2158. }
  2159. ///
  2160. /// @ingroup ge
  2161. /// @brief return not output to upper layer for cloud case
  2162. /// @param [in] data_id
  2163. /// @return Status result
  2164. ///
  2165. Status DavinciModel::ReturnNoOutput(uint32_t data_id) {
  2166. GELOGI("ReturnNoOutput model id:%u", model_id_);
  2167. for (auto op_desc : variable_op_list_) {
  2168. Status ret = VarManager::Instance(session_id_)
  2169. ->SyncBroadCastData2Var(runtime_param_.graph_id, op_desc->GetName(), op_desc, mem_base_);
  2170. GE_CHK_BOOL_EXEC(ret == SUCCESS, break, "sync var data ret failed, model id:%u, op name:%s.", model_id_,
  2171. op_desc->GetName().c_str());
  2172. }
  2173. GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null!");
  2174. std::vector<ge::OutputTensorInfo> outputs;
  2175. GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed.");
  2176. return SUCCESS;
  2177. }
  2178. void *DavinciModel::Run(DavinciModel *model) {
  2179. GE_CHK_BOOL_EXEC(model != nullptr,
  2180. CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
  2181. return nullptr, "model_pointer is null!")
  2182. bool seq_end_flag = false;
  2183. uint32_t model_id = model->Id();
  2184. uint32_t device_id = model->GetDeviceId();
  2185. GELOGI("Model Run thread start, model_id:%u.", model_id);
  2186. rtError_t rt_ret = rtSetDevice(static_cast<int32_t>(device_id));
  2187. if (rt_ret != RT_ERROR_NONE) {
  2188. GELOGE(FAILED, "Model run rtsetdevice failed.");
  2189. return nullptr;
  2190. }
  2191. // DeviceReset before thread run finished!
  2192. GE_MAKE_GUARD(not_used_var, [&] { GE_CHK_RT(rtDeviceReset(device_id)); });
  2193. while (model->RunFlag()) {
  2194. bool rslt_flg = true;
  2195. if (model->GetDataInputer() == nullptr) {
  2196. GELOGW("Data inputer is nullptr.");
  2197. CsaInteract::GetInstance().StoreInternalErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
  2198. break;
  2199. }
  2200. std::shared_ptr<InputDataWrapper> data_wrapper;
  2201. Status ret = model->GetDataInputer()->Pop(data_wrapper);
  2202. if (data_wrapper == nullptr || ret != SUCCESS) {
  2203. GELOGI("data_wrapper is null!");
  2204. continue;
  2205. }
  2206. GELOGI("Getting the input data, model_id:%u", model_id);
  2207. GE_IF_BOOL_EXEC(!model->RunFlag(), break);
  2208. InputData current_data = data_wrapper->GetInput();
  2209. GELOGI("Model thread Run begin, model id:%u, data index:%u.", model_id, current_data.index);
  2210. GE_TIMESTAMP_START(Model_SyncVarData);
  2211. ret = model->SyncVarData();
  2212. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  2213. ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
  2214. CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
  2215. continue, "Copy input data to model failed."); // [No need to check value]
  2216. GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(Model_SyncVarData, "Model Run SyncVarData"));
  2217. GELOGI("Copy input data, model id:%u", model_id);
  2218. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(),
  2219. model->SetProfileTime(MODEL_PRE_PROC_START));
  2220. ret = model->CopyInputData(current_data, false);
  2221. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
  2222. ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
  2223. CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
  2224. continue, "Copy input data to model failed."); // [No need to check value]
  2225. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END));
  2226. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START));
  2227. if (ProfilingManager::Instance().ProfilingOpTraceOn()) {
  2228. GELOGI("GetOpTraceIterNum:%d", ProfilingManager::Instance().GetOpTraceIterNum());
  2229. for (int32_t i = 0; i < ProfilingManager::Instance().GetOpTraceIterNum(); i++) {
  2230. if (!ProfilingManager::Instance().ProfilingLoadFlag()) {
  2231. vector<int32_t> prof_device_id_vec = ProfilingManager::Instance().GetProfilingDeviceId();
  2232. for (size_t j = 0; j < prof_device_id_vec.size(); ++j) {
  2233. // just profiling, no need to check value
  2234. (void)ProfilingManager::Instance().StartProfiling(i, prof_device_id_vec[j]);
  2235. }
  2236. }
  2237. GELOGI("rtModelExecute start.");
  2238. rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0);
  2239. GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false;
  2240. (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
  2241. continue); // [No need to check value]
  2242. GELOGI("rtModelExecute end");
  2243. GELOGI("rtStreamSynchronize start.");
  2244. rt_ret = rtStreamSynchronize(model->rt_model_stream_);
  2245. GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false;
  2246. (void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput());
  2247. continue); // [No need to check value]
  2248. GELOGI("rtStreamSynchronize end.");
  2249. (void)ProfilingManager::Instance().StopProfiling(); // just profiling, no need to check value
  2250. }
  2251. } else {
  2252. GE_TIMESTAMP_START(rtModelExecute);
  2253. GELOGI("rtModelExecute start.");
  2254. rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0);
  2255. GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false;
  2256. (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput());
  2257. CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC);
  2258. continue);
  2259. GELOGI("rtModelExecute end");
  2260. GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute"));
  2261. GE_TIMESTAMP_START(rtStreamSynchronize);
  2262. GELOGI("rtStreamSynchronize start.");
  2263. rt_ret = rtStreamSynchronize(model->rt_model_stream_);
  2264. if (rt_ret == RT_ERROR_END_OF_SEQUENCE) {
  2265. seq_end_flag = true;
  2266. }
  2267. GE_IF_BOOL_EXEC(
  2268. rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag);
  2269. (void)model->ReturnResult(current_data.index, false, seq_end_flag,
  2270. data_wrapper->GetOutput()); // [No need to check value]
  2271. CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC);
  2272. continue);
  2273. GELOGI("rtStreamSynchronize end.");
  2274. GE_IF_BOOL_EXEC(model->is_first_execute_,
  2275. GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize"));
  2276. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END));
  2277. }
  2278. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(),
  2279. model->SetProfileTime(MODEL_AFTER_PROC_START));
  2280. GE_TIMESTAMP_START(ReturnResult3);
  2281. // copy output data from device to host
  2282. GE_IF_BOOL_EXEC(!model->output_op_list_.empty(),
  2283. (void)model->ReturnResult(current_data.index, rslt_flg, false, data_wrapper->GetOutput()))
  2284. // copy output data from device to host for variable graph
  2285. GE_IF_BOOL_EXEC(model->output_op_list_.empty(), (void)model->ReturnNoOutput(current_data.index));
  2286. GE_IF_BOOL_EXEC(model->is_first_execute_,
  2287. GE_TIMESTAMP_EVENT_END(ReturnResult3, "GraphExcute::CopyDataFromDeviceToHost"));
  2288. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(),
  2289. model->SetProfileTime(MODEL_AFTER_PROC_END));
  2290. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), (void)model->SinkTimeProfile(current_data));
  2291. model->iterator_count_++;
  2292. model->is_first_execute_ = false;
  2293. GELOGI("run iterator count is %lu", model->iterator_count_);
  2294. }
  2295. CsaInteract::GetInstance().WriteInternalErrorCode();
  2296. GELOGI("Model run end, model id:%u", model->model_id_);
  2297. return nullptr;
  2298. }
  2299. ///
  2300. /// @ingroup ge
  2301. /// @brief call API provided by data inputer to destroy thread
  2302. /// @param [in] no
  2303. /// @return Status Destroy result
  2304. /// @author
  2305. ///
  2306. Status DavinciModel::DestroyThread() {
  2307. GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, "data_inputer_ is nullptr.");
  2308. run_flg_ = false;
  2309. data_inputer_->Stop();
  2310. if (thread_id_.joinable()) {
  2311. thread_id_.join();
  2312. }
  2313. return SUCCESS;
  2314. }
  2315. ///
  2316. /// @ingroup ge
  2317. /// @brief create model std::thread,
  2318. /// @brief start to execute Model
  2319. /// @param [in] no
  2320. /// @return Status create model thread and execute result
  2321. /// @author
  2322. ///
  2323. Status DavinciModel::ModelRunStart() {
  2324. GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, "data_inputer_ is nullptr.");
  2325. LockRunFlg();
  2326. GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); });
  2327. GE_CHK_BOOL_RET_STATUS(!run_flg_, INTERNAL_ERROR, "Model already started.");
  2328. run_flg_ = true;
  2329. // create stream instance which rt_model_handel is running on
  2330. GE_CHK_RT_RET(rtStreamCreate(&rt_model_stream_, priority_));
  2331. is_inner_model_stream_ = true;
  2332. string opt = "0";
  2333. (void)ge::GetContext().GetOption(OPTION_GE_MAX_DUMP_OP_NUM, opt); // option may not be set up, no need to check value
  2334. int64_t maxDumpOpNum = std::strtol(opt.c_str(), nullptr, kDecimal);
  2335. maxDumpOpNum_ = maxDumpOpNum;
  2336. CREATE_STD_THREAD(thread_id_, DavinciModel::Run, this);
  2337. GELOGI("model tread create success, model id:%u.", model_id_);
  2338. return SUCCESS;
  2339. }
  2340. ///
  2341. /// @ingroup ge
  2342. /// @brief call API provided by data inputer and destroy model Thread
  2343. /// @param [in] no
  2344. /// @return Status Destroy result
  2345. /// @author
  2346. ///
  2347. Status DavinciModel::ModelRunStop() {
  2348. LockRunFlg();
  2349. GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); });
  2350. GE_IF_BOOL_EXEC(!run_flg_, return SUCCESS);
  2351. GE_CHK_STATUS_RET(DestroyThread(), "DestoyThead failed.");
  2352. return SUCCESS;
  2353. }
  2354. void DavinciModel::UnbindTaskSinkStream() {
  2355. // unbinding hcom stream
  2356. UnbindHcomStream();
  2357. if (is_stream_list_bind_) {
  2358. for (size_t i = 0; i < stream_list_.size(); i++) {
  2359. // unbind rt_model_handle and streams
  2360. GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, stream_list_[i]) != RT_ERROR_NONE,
  2361. "Unbind stream from model failed! Index: %zu", i);
  2362. }
  2363. }
  2364. if (is_inner_model_stream_) {
  2365. if (!input_queue_ids_.empty() || !output_queue_ids_.empty()) {
  2366. GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_model_stream_) != RT_ERROR_NONE, "Unbind stream failed!");
  2367. }
  2368. // destroy stream that is bound with rt_model
  2369. GE_LOGW_IF(rtStreamDestroy(rt_model_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed.")
  2370. }
  2371. if (is_pure_head_stream_ && rt_head_stream_ != nullptr) {
  2372. GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_head_stream_) != RT_ERROR_NONE, "Unbind stream failed!");
  2373. GE_LOGW_IF(rtStreamDestroy(rt_head_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed.");
  2374. rt_head_stream_ = nullptr;
  2375. }
  2376. if (rt_entry_stream_ != nullptr) {
  2377. GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, rt_entry_stream_) != RT_ERROR_NONE, "Unbind stream failed!");
  2378. GE_LOGW_IF(rtStreamDestroy(rt_entry_stream_) != RT_ERROR_NONE, "Destroy stream for rt_model failed.");
  2379. rt_entry_stream_ = nullptr;
  2380. }
  2381. }
  2382. Status DavinciModel::CreateKnownZeroCopyMap(const vector<void *> &inputs, const vector<void *> &outputs) {
  2383. GELOGI("DavinciModel::CreateKnownZeroCopyMap in.");
  2384. if (inputs.size() > data_op_list_.size()) {
  2385. GELOGE(FAILED, "input data addr %u should less than input op number %u.", inputs.size(), data_op_list_.size());
  2386. return FAILED;
  2387. }
  2388. // remove zero copy addr in last iteration
  2389. knonw_input_data_info_.clear();
  2390. knonw_output_data_info_.clear();
  2391. for (size_t i = 0; i < inputs.size(); ++i) {
  2392. const vector<void *> addr_list = ModelUtils::GetOutputDataAddrs(runtime_param_, data_op_list_[i]);
  2393. knonw_input_data_info_[addr_list[kDataIndex]] = inputs[i];
  2394. GELOGI("DavinciModel::CreateKnownZeroCopyMap input %d,v addr %p,p addr %p .", i, addr_list[kDataIndex], inputs[i]);
  2395. }
  2396. if (output_op_list_.size() < kOutputNum) {
  2397. GELOGW("output op num in graph is %u.", output_op_list_.size());
  2398. return SUCCESS;
  2399. }
  2400. const vector<void *> addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, output_op_list_[kDataIndex]);
  2401. for (size_t i = 0; i < addr_list.size() && i < outputs.size(); ++i) {
  2402. knonw_output_data_info_[addr_list[i]] = outputs[i];
  2403. GELOGI("DavinciModel::CreateKnownZeroCopyMap output %d,v addr %p,p addr %p .", i, addr_list[i], outputs[i]);
  2404. }
  2405. GELOGI("DavinciModel::CreateKnownZeroCopyMap success.");
  2406. return SUCCESS;
  2407. }
  2408. Status DavinciModel::UpdateKnownZeroCopyAddr() {
  2409. for (size_t i = 0; i < total_io_addrs_.size(); ++i) {
  2410. auto it_in = knonw_input_data_info_.find(total_io_addrs_[i]);
  2411. if (it_in != knonw_input_data_info_.end()) {
  2412. GELOGI("DavinciModel::UpdateKnownZeroCopyAddr input %d,v addr %p,p addr %p .", i, total_io_addrs_[i],
  2413. knonw_input_data_info_.at(total_io_addrs_[i]));
  2414. total_io_addrs_[i] = knonw_input_data_info_.at(total_io_addrs_[i]);
  2415. }
  2416. auto it_out = knonw_output_data_info_.find(total_io_addrs_[i]);
  2417. if (it_out != knonw_output_data_info_.end()) {
  2418. GELOGI("DavinciModel::UpdateKnownZeroCopyAddr output %d,v addr %p,p addr %p .", i, total_io_addrs_[i],
  2419. knonw_output_data_info_.at(total_io_addrs_[i]));
  2420. total_io_addrs_[i] = knonw_output_data_info_.at(total_io_addrs_[i]);
  2421. }
  2422. }
  2423. GELOGI("DavinciModel::UpdateKnownZeroCopyAddr success.");
  2424. return SUCCESS;
  2425. }
  2426. Status DavinciModel::UpdateKnownNodeArgs(const vector<void *> &inputs, const vector<void *> &outputs) {
  2427. GELOGI("DavinciModel::UpdateKnownNodeArgs in");
  2428. GE_CHK_STATUS_RET(CreateKnownZeroCopyMap(inputs, outputs),
  2429. "DavinciModel::UpdateKnownNodeArgs create map for input/output zero copy.");
  2430. if (!base_addr_not_changed_) {
  2431. total_io_addrs_.clear();
  2432. orig_total_io_addrs_.clear();
  2433. for (size_t task_index = 0; task_index < task_list_.size(); ++task_index) {
  2434. auto &task = task_list_[task_index];
  2435. if (task != nullptr) {
  2436. Status ret = task->UpdateArgs();
  2437. if (ret != SUCCESS) {
  2438. GELOGE(FAILED, "task %d created by davinci model is nullptr.", task_index);
  2439. return FAILED;
  2440. }
  2441. }
  2442. }
  2443. // cache latest iterator io addr
  2444. orig_total_io_addrs_ = total_io_addrs_;
  2445. } else {
  2446. total_io_addrs_ = orig_total_io_addrs_;
  2447. }
  2448. GE_CHK_STATUS_RET(UpdateKnownZeroCopyAddr(), "DavinciModel::UpdateKnownZeroCopyAddr failed.");
  2449. if (total_args_size_ == 0) {
  2450. GELOGW("DavinciModel::UpdateKnownNodeArgs device args %p, dst size %u, pass rtMemcpy.", args_, total_args_size_);
  2451. } else {
  2452. uint32_t total_addr_size = total_io_addrs_.size() * sizeof(uint64_t);
  2453. GELOGI("DavinciModel::UpdateKnownNodeArgs device args %p, dst size %u, src size %u", args_, total_args_size_,
  2454. total_addr_size);
  2455. Status rt_ret =
  2456. rtMemcpy(args_, total_args_size_, total_io_addrs_.data(), total_addr_size, RT_MEMCPY_HOST_TO_DEVICE);
  2457. GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(rt_ret, "rtMemcpy error, ret: Ox%X", rt_ret); return FAILED;)
  2458. }
  2459. GELOGI("DavinciModel::UpdateKnownNodeArgs success");
  2460. return SUCCESS;
  2461. }
  2462. Status DavinciModel::InitTaskInfo(domi::ModelTaskDef &model_task_def) {
  2463. GELOGI("InitTaskInfo in, task size %zu", model_task_def.task().size());
  2464. task_list_.resize(model_task_def.task_size());
  2465. for (int i = 0; i < model_task_def.task_size(); ++i) {
  2466. // dynamic shape will create task_list_ before
  2467. const domi::TaskDef &task = model_task_def.task(i);
  2468. if (this->task_list_[i] == nullptr) {
  2469. task_list_[i] = TaskInfoFactory::Instance().Create(static_cast<rtModelTaskType_t>(task.type()));
  2470. }
  2471. GE_CHECK_NOTNULL(task_list_[i]);
  2472. Status ret = task_list_[i]->Init(task, this);
  2473. if (ret != SUCCESS) {
  2474. GELOGE(ret, "Task index %d init failed.", i);
  2475. return ret;
  2476. }
  2477. }
  2478. GELOGI("InitTaskInfo out");
  2479. return SUCCESS;
  2480. }
  2481. Status DavinciModel::MallocKnownArgs() {
  2482. GELOGI("DavinciModel::MallocKnownArgs in");
  2483. const auto &model_task_def = ge_model_->GetModelTaskDefPtr();
  2484. if (model_task_def->task_size() == 0) {
  2485. GELOGW("DavinciModel::MallocKnownArgs davincimodel has no task info.");
  2486. return SUCCESS;
  2487. }
  2488. task_list_.resize(model_task_def->task_size());
  2489. for (int32_t i = 0; i < model_task_def->task_size(); ++i) {
  2490. const domi::TaskDef &taskdef = model_task_def->task(i);
  2491. task_list_[i] = TaskInfoFactory::Instance().Create(static_cast<rtModelTaskType_t>(taskdef.type()));
  2492. GE_CHECK_NOTNULL(task_list_[i]);
  2493. Status ret = task_list_[i]->CalculateArgs(taskdef, this);
  2494. if (ret != SUCCESS) {
  2495. GELOGE(ret, "TaskInfo CalculateArgs failed.");
  2496. return ret;
  2497. }
  2498. }
  2499. // malloc args memory
  2500. if (total_args_size_ == 0) {
  2501. GELOGW("DavinciModel::MallocKnownArgs total_args_size_ equals to zero.");
  2502. return SUCCESS;
  2503. }
  2504. rtError_t rt_ret = rtMalloc(&args_, total_args_size_, RT_MEMORY_HBM);
  2505. if (rt_ret != RT_ERROR_NONE) {
  2506. GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret);
  2507. return RT_ERROR_TO_GE_STATUS(rt_ret);
  2508. }
  2509. // malloc fixed addr memory, eg: rts op
  2510. if (total_fixed_addr_size_ != 0) {
  2511. GELOGI("Begin to allocate fixed addr.");
  2512. rt_ret = rtMalloc(&fixed_addrs_, total_fixed_addr_size_, RT_MEMORY_HBM);
  2513. if (rt_ret != RT_ERROR_NONE) {
  2514. GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret);
  2515. return RT_ERROR_TO_GE_STATUS(rt_ret);
  2516. }
  2517. }
  2518. GELOGI("DavinciModel::MallocKnownArgs success, total args size %u. total fixed addr size %ld", total_args_size_,
  2519. total_fixed_addr_size_);
  2520. return SUCCESS;
  2521. }
  2522. Status DavinciModel::DistributeTask() {
  2523. GELOGI("do Distribute.");
  2524. for (auto &task : cpu_task_list_) {
  2525. if (task == nullptr) {
  2526. GELOGW("task is null");
  2527. continue;
  2528. }
  2529. GE_CHK_STATUS_RET(task->Distribute());
  2530. }
  2531. task_desc_info_.clear();
  2532. bool flag = GetL1FusionEnableOption();
  2533. char *skt_enable_env = std::getenv("SKT_ENABLE");
  2534. int64_t env_flag = (skt_enable_env != nullptr) ? std::strtol(skt_enable_env, nullptr, kDecimal) : 0;
  2535. if (env_flag != 0) {
  2536. flag = true;
  2537. }
  2538. const auto &model_task_def = ge_model_->GetModelTaskDefPtr();
  2539. GELOGI("there are %zu task need to save.", task_list_.size());
  2540. for (size_t task_index = 0; task_index < task_list_.size(); ++task_index) {
  2541. auto &task = task_list_.at(task_index);
  2542. GE_CHK_STATUS_RET(task->Distribute(), "Task[%zu] distribute fail", task_index);
  2543. // for data dump
  2544. auto op_index = std::max(model_task_def->task(task_index).kernel().context().op_index(),
  2545. model_task_def->task(task_index).kernel_ex().op_index());
  2546. OpDescPtr op = GetOpByIndex(op_index);
  2547. GE_CHECK_NOTNULL(op);
  2548. SaveDumpOpInfo(runtime_param_, op, task->GetTaskID(), task->GetStreamId());
  2549. if (reinterpret_cast<void *>(task->GetDumpArgs()) != nullptr) {
  2550. bool call_dump = GetDumpProperties().IsLayerNeedDump(name_, om_name_, op->GetName()) && task->CallSaveDumpInfo();
  2551. if (call_dump || is_op_debug_reg_) {
  2552. SaveDumpTask(task->GetTaskID(), task->GetStreamId(), op, task->GetDumpArgs());
  2553. }
  2554. }
  2555. // get op_name by task_index
  2556. if (task->GetCtx() != nullptr) {
  2557. auto iter = op_name_map_.find(task_index);
  2558. if (iter == op_name_map_.end()) {
  2559. continue;
  2560. }
  2561. // else task index is found in op_name_map_
  2562. TaskDescInfo task_desc_info;
  2563. string op_name = op_name_map_[task_index];
  2564. if (!om_name_.empty()) {
  2565. task_desc_info.model_name = om_name_;
  2566. } else {
  2567. task_desc_info.model_name = name_;
  2568. }
  2569. task_desc_info.op_name = op_name;
  2570. task_desc_info.block_dim = model_task_def->task(task_index).kernel().block_dim();
  2571. task_desc_info.task_id = task->GetTaskID();
  2572. task_desc_info.stream_id = task->GetStreamId();
  2573. task_desc_info_.emplace_back(task_desc_info);
  2574. if (flag) {
  2575. if (task->GetSktTaskID() != 0xFFFFFFFF) {
  2576. TaskDescInfo task_desc_info;
  2577. string op_name = "super_kernel_" + to_string(task_index);
  2578. task_desc_info.op_name = op_name;
  2579. task_desc_info.task_id = task->GetSktTaskID();
  2580. task_desc_info_.emplace_back(task_desc_info);
  2581. }
  2582. }
  2583. }
  2584. }
  2585. // launch dump kernel to aicpu
  2586. GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "Load dump info failed.");
  2587. return SUCCESS;
  2588. }
  2589. void DavinciModel::SetEndGraphId(uint32_t task_id, uint32_t stream_id) {
  2590. auto all_dump_model = GetDumpProperties().GetAllDumpModel();
  2591. bool findByOmName = all_dump_model.find(om_name_) != all_dump_model.end();
  2592. bool findByModelName = all_dump_model.find(name_) != all_dump_model.end();
  2593. if (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end() || findByOmName || findByModelName) {
  2594. GELOGI("start save end_graph_info to dumper, task_id is %u, stream_id is %u", task_id, stream_id);
  2595. data_dumper_.SaveEndGraphId(task_id, stream_id);
  2596. }
  2597. }
  2598. ///
  2599. /// @ingroup ge
  2600. /// @brief Set copy only for No task feed NetOutput address.
  2601. /// @return None.
  2602. ///
  2603. void DavinciModel::SetCopyOnlyOutput() {
  2604. for (const auto &output_outside_addrs : new_output_outside_addrs_) {
  2605. ZeroCopyOffset output_outside = output_outside_addrs.second;
  2606. for (uint32_t out_count = 0; out_count < output_outside.GetAddrCount(); ++out_count) {
  2607. auto &addrs_mapping_list = output_outside.GetOutsideAddrs();
  2608. std::map<const void *, std::vector<void *>> virtual_args_addrs = addrs_mapping_list[out_count];
  2609. for (const auto &virtual_args_addr : virtual_args_addrs) {
  2610. const auto &args_addrs = virtual_args_addr.second;
  2611. if (args_addrs.empty()) { // No task feed Output addr, Need copy directly.
  2612. GELOGI("[ZCPY] just copy %p to netoutput.", virtual_args_addr.first);
  2613. copy_only_addrs_.insert(virtual_args_addr.first);
  2614. }
  2615. }
  2616. }
  2617. }
  2618. }
  2619. ///
  2620. /// @ingroup ge
  2621. /// @brief Set disabled input zero copy addr.
  2622. /// @param [in] const void *addr: address of task
  2623. /// @return None.
  2624. ///
  2625. void DavinciModel::DisableZeroCopy(const void *addr) {
  2626. if (real_virtual_addrs_.find(addr) == real_virtual_addrs_.end()) {
  2627. return;
  2628. }
  2629. // Data link to RTS Op directly.
  2630. std::lock_guard<std::mutex> lock(outside_addrs_mutex_);
  2631. GELOGI("[ZCPY] disable zero copy of %p.", addr);
  2632. copy_only_addrs_.insert(addr);
  2633. }
  2634. ///
  2635. /// @ingroup ge
  2636. /// @brief Save outside address used info for ZeroCopy.
  2637. /// @param [in] const OpDescPtr &op_desc: current op desc
  2638. /// @param [in] const std::vector<void *> &outside_addrs: address of task
  2639. /// @param [in] const void *info: task args
  2640. /// @param [in] const char *args: task args
  2641. /// @param [in] size_t size: size of task args
  2642. /// @param [in] size_t offset: offset of task args
  2643. /// @return None.
  2644. ///
  2645. void DavinciModel::SetZeroCopyAddr(const OpDescPtr &op_desc, const std::vector<void *> &outside_addrs, const void *info,
  2646. void *args, size_t size, size_t offset) {
  2647. // Internal call has ensured that op_desc is not nullptr
  2648. GELOGD("[ZCPY] SetZeroCopyAddr for %s.", op_desc->GetName().c_str());
  2649. size_t nums = outside_addrs.size();
  2650. ZeroCopyTask zero_copy_task(op_desc->GetName(), static_cast<uint8_t *>(args), size);
  2651. for (size_t i = 0; i < nums; ++i) {
  2652. std::lock_guard<std::mutex> lock(outside_addrs_mutex_);
  2653. for (auto &input_outside_addrs : new_input_outside_addrs_) {
  2654. ZeroCopyOffset &input_outside = input_outside_addrs.second;
  2655. bool ret = input_outside.SetOutsideAddrsValue(zero_copy_task, outside_addrs[i], args, offset + i * kAddrLen);
  2656. if (ret) {
  2657. void *args_val = static_cast<uint8_t *>(args) + offset + i * kAddrLen;
  2658. SetBatchLabelAddr(op_desc, reinterpret_cast<uintptr_t>(args_val));
  2659. }
  2660. }
  2661. for (auto &output_outside_addrs : new_output_outside_addrs_) {
  2662. ZeroCopyOffset &output_outside = output_outside_addrs.second;
  2663. bool ret = output_outside.SetOutsideAddrsValue(zero_copy_task, outside_addrs[i], args, offset + i * kAddrLen);
  2664. if (ret) {
  2665. void *args_val = static_cast<uint8_t *>(args) + offset + i * kAddrLen;
  2666. SetBatchLabelAddr(op_desc, reinterpret_cast<uintptr_t>(args_val));
  2667. }
  2668. }
  2669. }
  2670. auto it = zero_copy_op_id_batch_label_.find(op_desc->GetId());
  2671. if (it == zero_copy_op_id_batch_label_.end()) {
  2672. zero_copy_task.SetBatchLabel(kDefaultBatchLable);
  2673. } else {
  2674. zero_copy_task.SetBatchLabel(it->second);
  2675. }
  2676. std::lock_guard<std::mutex> lock(outside_addrs_mutex_);
  2677. if (zero_copy_task.IsTaskArgsSet()) {
  2678. zero_copy_task.SetOriginalArgs(info, offset + nums * kAddrLen);
  2679. zero_copy_tasks_.emplace_back(zero_copy_task);
  2680. }
  2681. }
  2682. void DavinciModel::SetBatchLabelAddr(const OpDescPtr &op_desc, uintptr_t addr) {
  2683. // Establish a mapping between batch label and zero copy address for multi-batch scenes
  2684. auto it = zero_copy_op_id_batch_label_.find(op_desc->GetId());
  2685. if (it == zero_copy_op_id_batch_label_.end()) {
  2686. return;
  2687. }
  2688. const string &batch_label = it->second;
  2689. auto iter = zero_copy_batch_label_addrs_.find(batch_label);
  2690. if (iter != zero_copy_batch_label_addrs_.end()) {
  2691. iter->second.insert(addr);
  2692. GELOGD("[ZCPY] Set zero copy batch label and addrs success, batch label: %s, op name:%s.", batch_label.c_str(),
  2693. op_desc->GetName().c_str());
  2694. } else {
  2695. set<uintptr_t> addrs = {addr};
  2696. zero_copy_batch_label_addrs_.emplace(pair<string, set<uintptr_t>>(batch_label, addrs));
  2697. GELOGD("[ZCPY] New added zero copy batch label and addrs success, batch label: %s, op name:%s.",
  2698. batch_label.c_str(), op_desc->GetName().c_str());
  2699. }
  2700. }
  2701. ///
  2702. /// @ingroup ge
  2703. /// @brief Copy Check input size and model op size.
  2704. /// @param [in] const int64_t &input_size: input size.
  2705. /// @param [in] const int64_t &op_size: model op size.
  2706. /// @param [in] is_dynamic: dynamic batch input flag.
  2707. /// @return true if success
  2708. ///
  2709. bool DavinciModel::CheckInputAndModelSize(const int64_t &input_size, const int64_t &op_size, bool is_dynamic) {
  2710. if (is_dynamic) { // dynamic is max size.
  2711. GELOGI("No need to check input and model size.");
  2712. return true;
  2713. }
  2714. if (input_size > op_size) {
  2715. GELOGW(
  2716. "Input size [%u] is bigger than om size need [%u], "
  2717. "MAY cause inference result ERROR, please check model input",
  2718. input_size, op_size);
  2719. }
  2720. bool is_dynamic_aipp = false;
  2721. for (const auto &op_desc : data_op_list_) {
  2722. if (op_desc->GetType() == AIPP_DATA_TYPE) {
  2723. GELOGI("This is dynamic aipp model.");
  2724. is_dynamic_aipp = true;
  2725. break;
  2726. }
  2727. }
  2728. if (is_dynamic_aipp) {
  2729. GELOGI("This is dynamic aipp model, no need to judge smaller input size");
  2730. return true;
  2731. }
  2732. // Judge overflow first
  2733. if (input_size > (INT64_MAX - kDataMemAlignSizeCompare)) {
  2734. GELOGI("The Input size [%ld] is smaller than model size [%ld] and is in the range of 64 bytes", input_size,
  2735. op_size);
  2736. return true;
  2737. }
  2738. // The input and model input size can not be exactly equal because user input is not definite.
  2739. if ((input_size + kDataMemAlignSizeCompare) < op_size) {
  2740. GELOGE(FAILED, "Input size [%ld] can not be smaller than op size [%ld] after 64-byte alignment", input_size,
  2741. op_size);
  2742. return false;
  2743. }
  2744. return true;
  2745. }
  2746. ///
  2747. /// @ingroup ge
  2748. /// @brief Copy Inputs and Outputs addr to model for direct use.
  2749. /// @param [in] const InputData &input_data: model input data.
  2750. /// @param [in] OutputData &output_data: model output data.
  2751. /// @param [in] bool is_dynamic_input: whether is dynamic input, true: is dynamic input; false: not is dynamic input
  2752. /// @return SUCCESS handle successfully / PARAM_INVALID for failed
  2753. ///
  2754. Status DavinciModel::CopyModelData(const InputData &input_data, OutputData &output_data, bool is_dynamic) {
  2755. if (UpdateIoTaskArgs(new_input_data_info_, true, input_data.blobs, is_dynamic, input_data.batch_label) != SUCCESS) {
  2756. GELOGE(PARAM_INVALID, "[ZCPY] Update input data to model failed.");
  2757. return PARAM_INVALID;
  2758. }
  2759. if (UpdateIoTaskArgs(new_output_data_info_, false, output_data.blobs, is_dynamic, input_data.batch_label) !=
  2760. SUCCESS) {
  2761. GELOGE(PARAM_INVALID, "[ZCPY] Update output data to model failed.");
  2762. return PARAM_INVALID;
  2763. }
  2764. for (ZeroCopyTask &task : zero_copy_tasks_) {
  2765. GE_CHK_STATUS_RET(task.DistributeParam(is_async_mode_, rt_model_stream_), "[ZCPY] Update args failed.");
  2766. }
  2767. output_data.index = input_data.index;
  2768. output_data.model_id = model_id_;
  2769. return SUCCESS;
  2770. }
  2771. ///
  2772. /// @ingroup ge
  2773. /// @brief Copy Data addr to model for direct use.
  2774. /// @param [in] data_info: model memory addr/size map { data_index, { tensor_size, tensor_addr } }.
  2775. /// @param [in] is_input: input data or output data
  2776. /// @param [in] blobs: user input/output data list.
  2777. /// @param [in] is_dynamic: whether is dynamic input, true: is dynamic input; false: not is dynamic input
  2778. /// @param [in] batch_label: batch label for multi-batch scenes
  2779. /// @return SUCCESS handle successfully / others handle failed
  2780. ///
  2781. Status DavinciModel::UpdateIoTaskArgs(const std::map<uint32_t, ZeroCopyOffset> &data_info, bool is_input,
  2782. const vector<DataBuffer> &blobs, bool is_dynamic, const string &batch_label) {
  2783. string input_or_output = "input";
  2784. is_input ? input_or_output = "input" : input_or_output = "output";
  2785. if (blobs.size() != data_info.size()) {
  2786. GELOGE(FAILED, "Verify %s data num failed: model requires %zu, but user actually feeds %zu",
  2787. input_or_output.c_str(), data_info.size(), blobs.size());
  2788. return FAILED;
  2789. }
  2790. for (const auto &data : data_info) {
  2791. if (data.first >= blobs.size()) { // check data index.
  2792. GELOGE(FAILED, "Verify %s data num failed: can not find No.%zu data, because user only feeds %zu",
  2793. input_or_output.c_str(), data.first, blobs.size());
  2794. return FAILED;
  2795. }
  2796. const DataBuffer &buffer = blobs[data.first]; // index of data.
  2797. if (buffer.data == nullptr) {
  2798. GELOGE(FAILED, "data_buf.data is nullptr, index=%u", data.first);
  2799. return FAILED;
  2800. }
  2801. if (!CheckInputAndModelSize(buffer.length, data.second.GetDataSize(), is_dynamic)) {
  2802. GELOGE(FAILED, "Check input size and model size failed");
  2803. return FAILED;
  2804. }
  2805. void *basic_addr = data.second.GetBasicAddr();
  2806. uint64_t data_size = data.second.GetDataSize();
  2807. if (copy_only_addrs_.count(basic_addr) > 0) {
  2808. if (is_input) {
  2809. GELOGI("[IMAS] Find addr %p need direct copy from user malloc input %p", basic_addr, buffer.data);
  2810. if (rtMemcpy(basic_addr, data_size, buffer.data, buffer.length, RT_MEMCPY_DEVICE_TO_DEVICE) != RT_ERROR_NONE) {
  2811. GELOGE(FAILED, "Non-zero copy data node copy failed");
  2812. return FAILED;
  2813. }
  2814. }
  2815. GELOGI("No need to exeucte zero copy task because this addr %p need direct copy.", basic_addr);
  2816. continue;
  2817. }
  2818. for (size_t count = 0; count < data.second.GetDataCount(); ++count) {
  2819. int64_t size = data.second.GetDataInfo().at(count).first;
  2820. void *addr = data.second.GetDataInfo().at(count).second;
  2821. void *buffer_addr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(buffer.data) +
  2822. data.second.GetRelativeOffset().at(count));
  2823. GELOGI("[ZCPY] Copy %s blobs_index %u, virtual_addr: %p, size: %ld, user_data_addr: %p", input_or_output.c_str(),
  2824. data.first, addr, size, buffer_addr);
  2825. // For input data, just copy for rts task.
  2826. for (ZeroCopyTask &task : zero_copy_tasks_) {
  2827. if (task.GetBatchLabel() != kDefaultBatchLable && task.GetBatchLabel() != batch_label) {
  2828. continue;
  2829. }
  2830. uintptr_t addr_val = reinterpret_cast<uintptr_t>(addr);
  2831. if (task.UpdateTaskParam(addr_val, buffer_addr, zero_copy_batch_label_addrs_, batch_label) != SUCCESS) {
  2832. return FAILED;
  2833. }
  2834. }
  2835. }
  2836. }
  2837. return SUCCESS;
  2838. }
  2839. ///
  2840. /// @ingroup ge
  2841. /// @brief get unique identification for op when load two or more models
  2842. /// @param [in] const OpDescPtr: current op.
  2843. /// @param [in] string identification: unique identification for current op.
  2844. /// @return SUCCESS handle successfully / others handle failed
  2845. ///
  2846. void DavinciModel::GetUniqueId(const OpDescPtr &op_desc, std::string &unique_identification) {
  2847. std::string session_graph_id;
  2848. GE_IF_BOOL_EXEC(AttrUtils::GetStr(*op_desc, ATTR_NAME_SESSION_GRAPH_ID, session_graph_id),
  2849. GELOGD("Get original type of session_graph_id."));
  2850. if (session_graph_id.empty()) {
  2851. return;
  2852. } else if (session_graph_id.find("-1") != string::npos) {
  2853. unique_identification = session_graph_id + "_" + to_string(model_id_);
  2854. } else {
  2855. unique_identification = session_graph_id;
  2856. }
  2857. }
  2858. ///
  2859. /// @ingroup ge
  2860. /// @brief For TVM Op, avoid Addr Reuse.
  2861. /// @return void*
  2862. ///
  2863. const char *DavinciModel::GetRegisterStub(const string &binfile, const string &session_graph_id) {
  2864. string binfile_key;
  2865. if (session_graph_id.empty()) {
  2866. binfile_key = binfile;
  2867. } else {
  2868. binfile_key = session_graph_id + "_" + binfile;
  2869. }
  2870. auto it = tvm_bin_kernel_.find(binfile_key);
  2871. if (it != tvm_bin_kernel_.end()) {
  2872. return it->c_str();
  2873. } else {
  2874. it = tvm_bin_kernel_.insert(tvm_bin_kernel_.end(), binfile_key);
  2875. return it->c_str();
  2876. }
  2877. }
  2878. ///
  2879. /// @ingroup ge
  2880. /// @brief Constant Op Init.
  2881. /// @return Status
  2882. ///
  2883. Status DavinciModel::InitConstant(const OpDescPtr &op_desc) {
  2884. auto v_weights = ModelUtils::GetWeights(op_desc);
  2885. auto v_output_size = ModelUtils::GetOutputSize(op_desc);
  2886. auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param_, op_desc);
  2887. GE_IF_BOOL_EXEC(v_weights.empty() || v_output_size.empty() || v_output_addr.empty(),
  2888. GELOGE(PARAM_INVALID, "const op:%s not set output", op_desc->GetName().c_str());
  2889. return PARAM_INVALID;);
  2890. GeTensor *tensor = const_cast<GeTensor *>(v_weights[0].get());
  2891. GE_IF_BOOL_EXEC(static_cast<size_t>(v_output_size[0]) < tensor->GetData().size(),
  2892. GELOGE(PARAM_INVALID, "output size:%u less than weight data size:%zu", v_output_size[0],
  2893. tensor->GetData().size());
  2894. return PARAM_INVALID;);
  2895. GE_IF_BOOL_EXEC(tensor->GetData().size() == 0, GELOGW("const op:%s has no weight data.", op_desc->GetName().c_str());
  2896. return SUCCESS;);
  2897. auto desc = tensor->GetTensorDesc();
  2898. if (desc.GetDataType() == DT_STRING) {
  2899. GeShape tensor_shape = desc.GetShape();
  2900. /// if tensor is a scaler, it's shape size if zero, according ge_tensor.cc.
  2901. /// the logic of GetShapeSize is wrong, the scaler tensor's GetShapeSize is zero
  2902. /// and that of unknown shape is zero too.
  2903. /// unknown shape will not appear here, so we can use zero judge a tensor is scaler or not
  2904. int64_t elem_num = tensor_shape.GetShapeSize();
  2905. if (elem_num == 0 && tensor_shape.GetDims().size() == 0) {
  2906. elem_num = 1;
  2907. }
  2908. uint64_t *buff = reinterpret_cast<uint64_t *>(tensor->MutableData().data());
  2909. GE_CHK_BOOL_RET_STATUS(ge::CheckInt64Uint32MulOverflow(elem_num, kBytes) == SUCCESS, FAILED,
  2910. "Shape size is invalid");
  2911. uint64_t offset = static_cast<uint64_t>(elem_num * kBytes);
  2912. uint64_t hbm_raw_data_base_addr =
  2913. reinterpret_cast<uint64_t>(reinterpret_cast<uintptr_t>(v_output_addr[0])) + offset;
  2914. for (int64_t i = elem_num - 1; i >= 0; --i) {
  2915. buff[i] = hbm_raw_data_base_addr + (buff[i] - buff[0]);
  2916. }
  2917. }
  2918. GELOGI("[IMAS]InitConstant memcpy graph_%u type[V] name[%s] output[%d] memaddr[%p] mem_size[%u] datasize[%zu]",
  2919. runtime_param_.graph_id, op_desc->GetName().c_str(), 0, v_output_addr[0], v_output_size[0],
  2920. tensor->GetData().size());
  2921. GE_CHK_RT_RET(rtMemcpy(v_output_addr[0], v_output_size[0], tensor->GetData().data(), tensor->GetData().size(),
  2922. RT_MEMCPY_HOST_TO_DEVICE));
  2923. return SUCCESS;
  2924. }
  2925. ///
  2926. /// @ingroup ge
  2927. /// @brief TVM Op Init.
  2928. /// @return Status
  2929. ///
  2930. Status DavinciModel::InitTbeHandle(const OpDescPtr &op_desc) {
  2931. auto kernel = ge_model_->GetTBEKernelStore().FindKernel(op_desc->GetName());
  2932. auto tbe_kernel = (kernel != nullptr) ? kernel : op_desc->TryGetExtAttr(OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr());
  2933. if (tbe_kernel == nullptr) {
  2934. GELOGE(INTERNAL_ERROR, "TBE: %s can't find tvm bin file!", op_desc->GetName().c_str());
  2935. return INTERNAL_ERROR;
  2936. }
  2937. std::string session_graph_model_id;
  2938. GetUniqueId(op_desc, session_graph_model_id);
  2939. const char *bin_file_key = GetRegisterStub(op_desc->GetName(), session_graph_model_id); // from set, always valid.
  2940. TBEHandleStore &kernel_store = TBEHandleStore::GetInstance();
  2941. std::lock_guard<std::mutex> lock(tvm_bin_mutex_);
  2942. if (rtQueryFunctionRegistered(bin_file_key) != RT_ERROR_NONE) {
  2943. void *bin_handle = nullptr;
  2944. if (!kernel_store.FindTBEHandle(bin_file_key, bin_handle)) {
  2945. GELOGI("TBE: can't find the kernel_name[%s] in HandleMap", bin_file_key);
  2946. rtDevBinary_t binary;
  2947. std::string json_string;
  2948. GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, TVM_ATTR_NAME_MAGIC, json_string),
  2949. GELOGI("Get original type of session_graph_id."));
  2950. if (json_string == "RT_DEV_BINARY_MAGIC_ELF_AICPU") {
  2951. binary.magic = RT_DEV_BINARY_MAGIC_ELF_AICPU;
  2952. } else if (json_string == "RT_DEV_BINARY_MAGIC_ELF") {
  2953. binary.magic = RT_DEV_BINARY_MAGIC_ELF;
  2954. } else if (json_string == "RT_DEV_BINARY_MAGIC_ELF_AIVEC") {
  2955. binary.magic = RT_DEV_BINARY_MAGIC_ELF_AIVEC;
  2956. } else {
  2957. GELOGE(PARAM_INVALID, "TBE: Invalid parameter magic number! json: %s", json_string.c_str());
  2958. return PARAM_INVALID;
  2959. }
  2960. binary.version = 0;
  2961. binary.data = tbe_kernel->GetBinData();
  2962. binary.length = tbe_kernel->GetBinDataSize();
  2963. GELOGI("TBE: binary.length: %lu", binary.length);
  2964. GE_CHK_RT_RET(rtDevBinaryRegister(&binary, &bin_handle));
  2965. std::string meta_data;
  2966. GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, TVM_ATTR_NAME_METADATA, meta_data),
  2967. GELOGI("Get original type of json_string"));
  2968. GELOGI("TBE: meta data: %s", meta_data.empty() ? "null" : meta_data.c_str());
  2969. GE_IF_BOOL_EXEC(!meta_data.empty(), GE_CHK_RT_RET(rtMetadataRegister(bin_handle, meta_data.c_str())));
  2970. kernel_store.StoreTBEHandle(bin_file_key, bin_handle, tbe_kernel);
  2971. } else {
  2972. GELOGI("TBE: find the kernel_name[%s] in HandleMap", bin_file_key);
  2973. kernel_store.ReferTBEHandle(bin_file_key);
  2974. }
  2975. std::string kernel_name;
  2976. GE_IF_BOOL_EXEC(AttrUtils::GetStr(op_desc, op_desc->GetName() + "_kernelname", kernel_name),
  2977. GELOGI("Get original type of kernel_name"));
  2978. GELOGI("TBE: binfile_key=%s, kernel_name=%s", bin_file_key, kernel_name.c_str());
  2979. GE_CHK_RT_RET(rtFunctionRegister(bin_handle, bin_file_key, bin_file_key, kernel_name.c_str(), 0));
  2980. used_tbe_handle_map_[bin_file_key] = 1; // Init used num to 1.
  2981. return SUCCESS;
  2982. }
  2983. // Kernel registed, Increase used num in store.
  2984. StoreTbeHandle(bin_file_key);
  2985. return SUCCESS;
  2986. }
  2987. void DavinciModel::StoreTbeHandle(const std::string &handle_key) {
  2988. // Online mode FE may call rtFunctionRegister.
  2989. TBEHandleStore &kernel_store = TBEHandleStore::GetInstance();
  2990. auto it = used_tbe_handle_map_.find(handle_key);
  2991. if (it != used_tbe_handle_map_.end()) {
  2992. // GE registered, increase reference.
  2993. kernel_store.ReferTBEHandle(handle_key);
  2994. it->second++;
  2995. return;
  2996. }
  2997. void *bin_handle = nullptr;
  2998. if (kernel_store.FindTBEHandle(handle_key, bin_handle)) {
  2999. // GE registered, increase reference.
  3000. used_tbe_handle_map_[handle_key] = 1; // Init used num to 1.
  3001. kernel_store.ReferTBEHandle(handle_key);
  3002. }
  3003. }
  3004. void DavinciModel::CleanTbeHandle() {
  3005. TBEHandleStore &kernel_store = TBEHandleStore::GetInstance();
  3006. kernel_store.EraseTBEHandle(used_tbe_handle_map_);
  3007. used_tbe_handle_map_.clear();
  3008. tvm_bin_kernel_.clear();
  3009. }
  3010. ///
  3011. /// @ingroup ge
  3012. /// @brief insert active_stream_indication_
  3013. /// @return Status
  3014. ///
  3015. Status DavinciModel::InitStreamActive(const OpDescPtr &op_desc) {
  3016. if (op_desc->HasAttr(ATTR_NAME_SWITCH_BRANCH_NODE_LABEL)) {
  3017. std::vector<uint32_t> active_stream_list;
  3018. GE_CHK_BOOL_EXEC(AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list),
  3019. return INTERNAL_ERROR, "StreamActiveOp get attr ACTIVE_STREAM failed.");
  3020. for (size_t j = 0; j < active_stream_list.size(); ++j) {
  3021. active_stream_indication_.insert(active_stream_list[j]);
  3022. GELOGI("flowctrl_op_index_map node:%s, active_stream_id=%u.", op_desc->GetName().c_str(), active_stream_list[j]);
  3023. }
  3024. }
  3025. return SUCCESS;
  3026. }
  3027. Status DavinciModel::InitStreamSwitch(const OpDescPtr &op_desc) {
  3028. std::vector<uint32_t> active_stream_list;
  3029. GE_LOGI_IF(!ge::AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list),
  3030. "GetInt ACTIVE_STREAM_LIST failed.");
  3031. if (active_stream_list.size() != kTrueBranchStreamNum) {
  3032. GELOGE(INTERNAL_ERROR, "Stream num of switch true branch must be %u.", kTrueBranchStreamNum);
  3033. return INTERNAL_ERROR;
  3034. }
  3035. uint32_t true_stream_id = active_stream_list.front();
  3036. active_stream_indication_.insert(true_stream_id);
  3037. GELOGI("flowctrl_op_index_map node:%s, true_stream_id=%u.", op_desc->GetName().c_str(), true_stream_id);
  3038. return SUCCESS;
  3039. }
  3040. Status DavinciModel::InitStreamSwitchN(const OpDescPtr &op_desc) {
  3041. std::vector<uint32_t> active_stream_list;
  3042. if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list)) {
  3043. GELOGE(INTERNAL_ERROR, "StreamSwitchNOp get attr ACTIVE_STREAM failed.");
  3044. return INTERNAL_ERROR;
  3045. }
  3046. for (size_t j = 0; j < active_stream_list.size(); ++j) {
  3047. active_stream_indication_.insert(active_stream_list[j]);
  3048. GELOGI("StreamSwitchNOp node:%s, active_stream_id=%u.", op_desc->GetName().c_str(), active_stream_list[j]);
  3049. }
  3050. uint32_t batch_num = 0;
  3051. if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, batch_num)) {
  3052. GELOGE(FAILED, "Failed to get attr ATTR_NAME_BATCH_NUM, StreamSwitchN: %s.", op_desc->GetName().c_str());
  3053. return FAILED;
  3054. }
  3055. return SetDynamicBatchInfo(op_desc, batch_num);
  3056. }
  3057. Status DavinciModel::SetDynamicBatchInfo(const OpDescPtr &op_desc, uint32_t batch_num) {
  3058. batch_info_.clear();
  3059. combined_batch_info_.clear();
  3060. (void)AttrUtils::GetInt(op_desc, ATTR_DYNAMIC_TYPE, dynamic_type_);
  3061. (void)AttrUtils::GetListStr(op_desc, ATTR_USER_DESIGNEATE_SHAPE_ORDER, user_designate_shape_order_);
  3062. for (uint32_t i = 0; i < batch_num; ++i) {
  3063. std::vector<int64_t> batch_shape;
  3064. const std::string attr_name = ATTR_NAME_PRED_VALUE + "_" + std::to_string(i);
  3065. if (!AttrUtils::GetListInt(op_desc, attr_name, batch_shape)) {
  3066. GELOGE(FAILED, "Get attr ATTR_NAME_PRED_VALUE failed, Node: %s", op_desc->GetName().c_str());
  3067. batch_info_.clear();
  3068. return FAILED;
  3069. }
  3070. batch_info_.emplace_back(batch_shape);
  3071. batch_shape.clear();
  3072. const string attr_combined_batch = ATTR_NAME_COMBINED_BATCH + "_" + std::to_string(i);
  3073. if (AttrUtils::GetListInt(op_desc, attr_combined_batch, batch_shape)) {
  3074. combined_batch_info_.emplace_back(batch_shape);
  3075. }
  3076. }
  3077. return SUCCESS;
  3078. }
  3079. Status DavinciModel::InitCase(const OpDescPtr &op_desc) {
  3080. uint32_t batch_num = 0;
  3081. if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, batch_num)) {
  3082. GELOGI("Not multi-batch Node: %s", op_desc->GetName().c_str());
  3083. return SUCCESS;
  3084. }
  3085. return SetDynamicBatchInfo(op_desc, batch_num);
  3086. }
  3087. bool DavinciModel::IsBroadCastOpData(const ge::NodePtr &var_node) {
  3088. for (auto out_anchor : var_node->GetAllOutDataAnchors()) {
  3089. GE_RT_FALSE_CHECK_NOTNULL(out_anchor);
  3090. for (auto in_anchor : out_anchor->GetPeerInDataAnchors()) {
  3091. GE_RT_FALSE_CHECK_NOTNULL(in_anchor);
  3092. ge::NodePtr dst_node = in_anchor->GetOwnerNode();
  3093. GE_RT_FALSE_CHECK_NOTNULL(dst_node);
  3094. if (dst_node->GetType() == HCOMBROADCAST || dst_node->GetType() == HVDCALLBACKBROADCAST) {
  3095. return true;
  3096. }
  3097. }
  3098. }
  3099. return false;
  3100. }
  3101. ///
  3102. /// @ingroup ge
  3103. /// @brief Init model stream for NN model.
  3104. /// @param [in] stream user input model stream.
  3105. /// @return Status
  3106. ///
  3107. Status DavinciModel::InitModelStream(rtStream_t stream) {
  3108. ExecuteMode curr_mode = is_async_mode_ ? ASYNCHRONIZATION : SYNCHRONIZATION;
  3109. GE_CHK_BOOL_RET_STATUS((curr_mode == last_execute_mode_) || (last_execute_mode_ == INITIALIZATION), INTERNAL_ERROR,
  3110. "NnExecute not support mix execute.");
  3111. last_execute_mode_ = curr_mode;
  3112. // asynchronize mode, use user input stream.
  3113. if (is_async_mode_) {
  3114. rt_model_stream_ = stream;
  3115. is_inner_model_stream_ = false;
  3116. return SUCCESS;
  3117. }
  3118. // synchronize mode, use forbidden stream.
  3119. if (stream != nullptr) {
  3120. if ((rt_model_stream_ != nullptr) && is_inner_model_stream_) {
  3121. GE_LOGW_IF(rtStreamDestroy(rt_model_stream_) != RT_ERROR_NONE, "Destroy rt_stream failed!");
  3122. }
  3123. rt_model_stream_ = stream;
  3124. is_inner_model_stream_ = false;
  3125. return SUCCESS;
  3126. }
  3127. if (rt_model_stream_ == nullptr) {
  3128. GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_model_stream_, priority_, RT_STREAM_FORBIDDEN_DEFAULT));
  3129. is_inner_model_stream_ = true;
  3130. }
  3131. return SUCCESS;
  3132. }
  3133. ///
  3134. /// @ingroup ge
  3135. /// @brief ACL case, do not start new thread, return execute result.
  3136. /// @param [in] stream execute model stream.
  3137. /// @param [in] async_mode is asynchronize mode.
  3138. /// @param [in] input_data model input data.
  3139. /// @param [out] output_data model output data.
  3140. ///
  3141. Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputData &input_data,
  3142. OutputData &output_data) {
  3143. is_async_mode_ = async_mode;
  3144. GELOGI("Model Run begin, model id:%u, data index:%u, flag:%d.", model_id_, input_data.index, is_async_mode_);
  3145. GE_CHK_STATUS_RET(InitModelStream(stream), "Init model stream failed.");
  3146. is_dynamic_ = input_data.is_dynamic_batch;
  3147. if (!is_dynamic_) {
  3148. zero_copy_batch_label_addrs_.clear();
  3149. }
  3150. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_PRE_PROC_START));
  3151. Status ret = CopyModelData(input_data, output_data, is_dynamic_);
  3152. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Copy input data to model failed. model id: %u",
  3153. model_id_);
  3154. GELOGI("current_data.index=%u", input_data.index);
  3155. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_PRE_PROC_END));
  3156. if (!task_list_.empty()) {
  3157. GELOGD("rtModelExecute do");
  3158. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_INFER_START));
  3159. rtError_t rt_ret = rtModelExecute(rt_model_handle_, rt_model_stream_, 0);
  3160. GE_CHK_RT_EXEC(rt_ret, return RT_ERROR_TO_GE_STATUS(rt_ret));
  3161. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_INFER_END));
  3162. GELOGI("rtModelExecute end");
  3163. }
  3164. if (!is_async_mode_) {
  3165. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_AFTER_PROC_START));
  3166. ret = CopyOutputData(input_data.index, output_data, RT_MEMCPY_DEVICE_TO_DEVICE);
  3167. GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Copy Output data to user failed.");
  3168. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), SetProfileTime(MODEL_AFTER_PROC_END));
  3169. }
  3170. // report model time data
  3171. GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), (void)SinkTimeProfile(input_data));
  3172. GELOGI("Model run end, model id:%u", model_id_);
  3173. return SUCCESS;
  3174. }
  3175. // Add active entry stream for special env.
  3176. Status DavinciModel::AddHeadStream() {
  3177. if (active_stream_list_.empty()) {
  3178. GELOGE(INTERNAL_ERROR, "Active stream is empty, stream list size: %zu, stream indication size: %zu.",
  3179. stream_list_.size(), active_stream_indication_.size());
  3180. return INTERNAL_ERROR;
  3181. }
  3182. if (active_stream_list_.size() == 1) {
  3183. GELOGI("Just one active stream, take as head stream.");
  3184. rt_head_stream_ = active_stream_list_[0];
  3185. is_pure_head_stream_ = false;
  3186. } else {
  3187. // Create stream which rt_model_handel running on, this is S0, TS stream.
  3188. GELOGI("Multiple active stream: %zu, create head stream.", active_stream_list_.size());
  3189. GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_head_stream_, priority_, RT_STREAM_PERSISTENT));
  3190. GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, rt_head_stream_, RT_INVALID_FLAG)); // Not active.
  3191. is_pure_head_stream_ = true;
  3192. for (auto s : active_stream_list_) {
  3193. std::shared_ptr<CpuTaskActiveEntry> active_entry = MakeShared<CpuTaskActiveEntry>(rt_head_stream_);
  3194. if (active_entry == nullptr) {
  3195. GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed.");
  3196. return MEMALLOC_FAILED;
  3197. }
  3198. Status status = active_entry->Init(s);
  3199. if (status != SUCCESS) {
  3200. return status;
  3201. }
  3202. cpu_task_list_.emplace_back(active_entry);
  3203. }
  3204. }
  3205. // Create entry stream active head stream. AICPU stream.
  3206. GE_CHK_RT_RET(rtStreamCreateWithFlags(&rt_entry_stream_, priority_, RT_STREAM_AICPU));
  3207. GE_CHK_RT_RET(rtModelBindStream(rt_model_handle_, rt_entry_stream_, RT_HEAD_STREAM));
  3208. return SUCCESS;
  3209. }
  3210. Status DavinciModel::InitEntryTask() {
  3211. if (deploy_type_ == AICPU_DEPLOY_CROSS_THREAD) {
  3212. GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed.");
  3213. return CpuActiveStream();
  3214. } else {
  3215. return LoadWithQueue();
  3216. }
  3217. }
  3218. uint8_t *DavinciModel::MallocFeatureMapMem(size_t data_size) {
  3219. uint8_t *mem_base = nullptr;
  3220. const string purpose("feature map,used for op input and output.");
  3221. if (std::getenv(kEnvGeuseStaticMemory) != nullptr) {
  3222. data_size = static_cast<size_t>(VarManager::Instance(session_id_)->GetGraphMemoryMaxSize());
  3223. string memory_key = std::to_string(0) + "_f";
  3224. mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, memory_key, data_size, GetDeviceId());
  3225. } else {
  3226. mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, data_size, GetDeviceId());
  3227. }
  3228. if (mem_base != nullptr) {
  3229. GE_CHK_RT(rtMemset(mem_base, data_size, 0U, data_size));
  3230. }
  3231. return mem_base;
  3232. }
  3233. uint8_t *DavinciModel::MallocP2PMem(size_t p2p_data_size) {
  3234. uint8_t *p2p_mem_base = nullptr;
  3235. const string purpose("p2p memory, used for some op related to hcom");
  3236. if (std::getenv(kEnvGeuseStaticMemory) != nullptr) {
  3237. string p2p_memory_key = std::to_string(0) + "_p";
  3238. p2p_mem_base =
  3239. MemManager::Instance(RT_MEMORY_P2P_DDR)->MallocMemory(purpose, p2p_memory_key, p2p_data_size, GetDeviceId());
  3240. } else {
  3241. p2p_mem_base = MemManager::Instance(RT_MEMORY_P2P_DDR)->MallocMemory(purpose, p2p_data_size, GetDeviceId());
  3242. }
  3243. return p2p_mem_base;
  3244. }
  3245. uint8_t *DavinciModel::MallocWeightsMem(size_t weights_size) {
  3246. uint8_t *weights_mem_base = nullptr;
  3247. const string purpose("weights memory in inference network.");
  3248. if (std::getenv(kEnvGeuseStaticMemory) != nullptr) {
  3249. string weight_memory_key = std::to_string(0) + "_w";
  3250. weights_mem_base =
  3251. MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, weight_memory_key, weights_size, GetDeviceId());
  3252. } else {
  3253. weights_mem_base = MemManager::Instance(RT_MEMORY_HBM)->MallocMemory(purpose, weights_size, GetDeviceId());
  3254. }
  3255. return weights_mem_base;
  3256. }
  3257. void DavinciModel::FreeFeatureMapMem() {
  3258. if (std::getenv(kEnvGeuseStaticMemory) != nullptr && is_inner_mem_base_) {
  3259. string weight_memory_key = std::to_string(0) + "_f";
  3260. if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(weight_memory_key) != nullptr) {
  3261. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weight_memory_key, GetDeviceId()),
  3262. "failed to free weight memory");
  3263. }
  3264. mem_base_ = nullptr;
  3265. } else {
  3266. GE_IF_BOOL_EXEC(mem_base_ != nullptr && is_inner_mem_base_,
  3267. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(mem_base_, GetDeviceId()),
  3268. "failed to free feature_map memory");
  3269. mem_base_ = nullptr);
  3270. }
  3271. }
  3272. void DavinciModel::FreeP2PMem() {
  3273. if (std::getenv(kEnvGeuseStaticMemory) != nullptr) {
  3274. std::string p2p_memory_key = std::to_string(0) + "_p";
  3275. if (MemManager::Instance(RT_MEMORY_P2P_DDR)->GetMemoryAddr(p2p_memory_key) != nullptr) {
  3276. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_memory_key, GetDeviceId()),
  3277. "failed to free p2p memory");
  3278. }
  3279. p2p_mem_base_ = nullptr;
  3280. } else {
  3281. GE_IF_BOOL_EXEC(p2p_mem_base_ != nullptr && is_inner_mem_base_,
  3282. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_mem_base_, GetDeviceId()),
  3283. "failed to free p2p memory");
  3284. p2p_mem_base_ = nullptr);
  3285. }
  3286. }
  3287. void DavinciModel::FreeWeightsMem() {
  3288. if (std::getenv(kEnvGeuseStaticMemory) != nullptr) {
  3289. string memory_key = std::to_string(0) + "_w";
  3290. if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(memory_key) != nullptr) {
  3291. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(memory_key, GetDeviceId()),
  3292. "failed to free feature_map memory");
  3293. }
  3294. weights_mem_base_ = nullptr;
  3295. } else {
  3296. GE_IF_BOOL_EXEC(weights_mem_base_ != nullptr && weights_mem_base_ != mem_base_ && is_inner_weight_base_,
  3297. GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weights_mem_base_, GetDeviceId()),
  3298. "failed to free weight memory");
  3299. weights_mem_base_ = nullptr);
  3300. }
  3301. }
  3302. Status DavinciModel::TransAllVarData(ComputeGraphPtr &graph, uint32_t graph_id) {
  3303. GELOGI("TransAllVarData start: session_id:%lu, graph_id: %u.", session_id_, graph_id);
  3304. rtContext_t ctx = nullptr;
  3305. rtError_t rt_ret = rtCtxGetCurrent(&ctx);
  3306. if (rt_ret != RT_ERROR_NONE) {
  3307. GELOGE(RT_FAILED, "Failed to get current context, error_code is: 0x%X.", rt_ret);
  3308. return RT_ERROR_TO_GE_STATUS(rt_ret);
  3309. }
  3310. std::vector<NodePtr> variable_node_list;
  3311. for (ge::NodePtr &node : graph->GetDirectNode()) {
  3312. if (node == nullptr) {
  3313. continue;
  3314. }
  3315. if (node->GetType() != VARIABLE) {
  3316. continue;
  3317. }
  3318. variable_node_list.emplace_back(node);
  3319. }
  3320. GE_CHK_STATUS_RET_NOLOG(
  3321. TransVarDataUtils::TransAllVarData(variable_node_list, session_id_, ctx, graph_id, kThreadNum));
  3322. GELOGI("TransAllVarData success.");
  3323. return SUCCESS;
  3324. }
  3325. void DavinciModel::SetDataDumperArgs(const ComputeGraphPtr &compute_graph) {
  3326. GELOGI("set data dumper args, name: %s, id: %u.", name_.c_str(), model_id_);
  3327. data_dumper_.SetModelName(name_);
  3328. data_dumper_.SetModelId(model_id_);
  3329. data_dumper_.SetOmName(om_name_);
  3330. data_dumper_.SetComputeGraph(compute_graph);
  3331. data_dumper_.SetRefInfo(saved_task_addrs_);
  3332. data_dumper_.SetL1FusionAddr(l1_fusion_addr_);
  3333. int32_t device_id = 0;
  3334. rtError_t rt_ret = rtGetDevice(&device_id);
  3335. if (rt_ret != RT_ERROR_NONE || device_id < 0) {
  3336. GELOGE(RT_FAILED, "Call rtGetDevice failed, ret = 0x%X, device_id = %d.", rt_ret, device_id);
  3337. return;
  3338. }
  3339. data_dumper_.SetDeviceId(device_id);
  3340. // set loop count addr
  3341. auto get_var_addr = [](const OpDescPtr &op, const RuntimeParam &runtime_param) -> void * {
  3342. if (op != nullptr) {
  3343. auto v_output_size = ModelUtils::GetOutputSize(op);
  3344. auto v_output_addr = ModelUtils::GetOutputDataAddrs(runtime_param, op);
  3345. if (v_output_size.empty() || v_output_addr.empty()) {
  3346. return nullptr;
  3347. }
  3348. return v_output_addr[0];
  3349. }
  3350. GELOGW("op is null.");
  3351. return nullptr;
  3352. };
  3353. data_dumper_.SetLoopAddr(get_var_addr(GetVariableOp(NODE_NAME_GLOBAL_STEP), runtime_param_),
  3354. get_var_addr(GetVariableOp(NODE_NAME_FLOWCTRL_LOOP_PER_ITER), runtime_param_),
  3355. get_var_addr(GetVariableOp(NODE_NAME_FLOWCTRL_LOOP_COND), runtime_param_));
  3356. GELOGI("SetDataDumperArgs end.");
  3357. }
  3358. uint32_t DavinciModel::GetFlowctrlIndex(uint32_t op_index) {
  3359. std::lock_guard<std::mutex> lock(flowctrl_op_index_internal_map_mutex_);
  3360. return (++flowctrl_op_index_internal_map_[op_index]) - 1;
  3361. }
  3362. void DavinciModel::PushHcclStream(rtStream_t value) {
  3363. std::lock_guard<std::mutex> lock(all_hccl_stream_list_mutex_);
  3364. all_hccl_stream_list_.push_back(value);
  3365. }
  3366. void DavinciModel::SaveHcclFollowStream(int64_t main_stream_id, rtStream_t stream) {
  3367. std::lock_guard<std::mutex> lock(capacity_of_stream_mutex_);
  3368. main_follow_stream_mapping_[main_stream_id].emplace_back(stream);
  3369. }
  3370. Status DavinciModel::GetComputeGraphInfo(const ComputeGraphPtr &graph, vector<ComputeGraphDescInfo> &graph_desc_info) {
  3371. GELOGI("GetComputeGraphInfo start.");
  3372. for (auto &node : graph->GetAllNodes()) {
  3373. ComputeGraphDescInfo compute_graph_info;
  3374. auto op_desc = node->GetOpDesc();
  3375. if (op_desc == nullptr) {
  3376. GELOGE(PARAM_INVALID, "op_desc is nullptr.");
  3377. return PARAM_INVALID;
  3378. }
  3379. auto op_mode = static_cast<uint32_t>(domi::ImplyType::INVALID);
  3380. if (AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, op_mode) &&
  3381. op_mode == static_cast<uint32_t>(domi::ImplyType::TVM)) {
  3382. if (!om_name_.empty()) {
  3383. compute_graph_info.model_name = om_name_;
  3384. } else {
  3385. compute_graph_info.model_name = name_;
  3386. }
  3387. compute_graph_info.op_name = op_desc->GetName();
  3388. compute_graph_info.op_type = op_desc->GetType();
  3389. for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) {
  3390. GeTensorDescPtr input_desc = op_desc->MutableInputDesc(i);
  3391. if (input_desc == nullptr) {
  3392. continue;
  3393. }
  3394. compute_graph_info.input_format.emplace_back(input_desc->GetFormat());
  3395. compute_graph_info.input_shape.emplace_back(input_desc->GetShape().GetDims());
  3396. compute_graph_info.input_data_type.emplace_back(input_desc->GetDataType());
  3397. }
  3398. for (size_t j = 0; j < op_desc->GetOutputsSize(); ++j) {
  3399. GeTensorDesc output_desc = op_desc->GetOutputDesc(j);
  3400. compute_graph_info.output_format.emplace_back(output_desc.GetFormat());
  3401. compute_graph_info.output_shape.emplace_back(output_desc.GetShape().GetDims());
  3402. compute_graph_info.output_data_type.emplace_back(output_desc.GetDataType());
  3403. }
  3404. graph_desc_info.emplace_back(compute_graph_info);
  3405. }
  3406. }
  3407. GELOGI("GetComputeGraphInfo end.");
  3408. return SUCCESS;
  3409. }
  3410. void DavinciModel::SetTotalFixedAddrsSize(string tensor_name, int64_t fix_addr_size) {
  3411. if (tensor_name_to_fixed_addr_size_.find(tensor_name) == tensor_name_to_fixed_addr_size_.end()) {
  3412. tensor_name_to_fixed_addr_size_[tensor_name] = total_fixed_addr_size_;
  3413. total_fixed_addr_size_ += fix_addr_size;
  3414. }
  3415. }
  3416. Status DavinciModel::GetOrigInputInfo(uint32_t index, OriginInputInfo &orig_input_info) {
  3417. GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index);
  3418. OpDescPtr data_op = data_op_list_[index];
  3419. if (!data_op->HasAttr(ATTR_NAME_AIPP_INPUTS) || !data_op->HasAttr(ATTR_NAME_AIPP_OUTPUTS)) {
  3420. GELOGE(GE_AIPP_NOT_EXIST, "GetOrigInputInfo: there is not AIPP related with index %u.", index);
  3421. return GE_AIPP_NOT_EXIST;
  3422. }
  3423. vector<std::string> inputs;
  3424. if (AttrUtils::GetListStr(data_op, ATTR_NAME_AIPP_INPUTS, inputs) && !inputs.empty()) {
  3425. std::string input = inputs[kAippOriginInputIndex];
  3426. GELOGI("GetOrigInputInfo: origin input str: %s", input.c_str());
  3427. std::vector<std::string> infos = ge::StringUtils::Split(input, ':');
  3428. if (infos.size() != kAippInfoNum) {
  3429. GELOGW("origin input str is invalid.");
  3430. }
  3431. orig_input_info.format = TypeUtils::SerialStringToFormat(infos[kAippInfoFormat]);
  3432. orig_input_info.data_type = TypeUtils::SerialStringToDataType(infos[kAippInfoDataType]);
  3433. orig_input_info.dim_num = std::strtol(infos[kAippInfoDimNum].c_str(), nullptr, kDecimal);
  3434. }
  3435. return SUCCESS;
  3436. }
  3437. void DavinciModel::ParseAIPPInfo(std::string in_out_info, InputOutputDims &dims_info) {
  3438. GELOGI("ParseAIPPInfo: origin str: %s", in_out_info.c_str());
  3439. std::vector<std::string> infos = ge::StringUtils::Split(in_out_info, ':');
  3440. if (infos.size() != kAippInfoNum) {
  3441. GELOGW("origin input str is invalid.");
  3442. }
  3443. dims_info.name = infos[kAippInfoTensorName];
  3444. dims_info.size = std::strtol(infos[kAippInfoTensorSize].c_str(), nullptr, kDecimal);
  3445. dims_info.dim_num = std::strtol(infos[kAippInfoDimNum].c_str(), nullptr, kDecimal);
  3446. std::vector<std::string> dims = ge::StringUtils::Split(infos[kAippInfoShape], ',');
  3447. for (const auto &dim : dims) {
  3448. if (dim.empty()) {
  3449. continue;
  3450. }
  3451. dims_info.dims.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal));
  3452. }
  3453. }
  3454. Status DavinciModel::GetAllAippInputOutputDims(uint32_t index, std::vector<InputOutputDims> &input_dims,
  3455. std::vector<InputOutputDims> &output_dims) {
  3456. GE_CHK_BOOL_RET_STATUS(index < data_op_list_.size(), PARAM_INVALID, "Index %u is invalid.", index);
  3457. OpDescPtr data_op = data_op_list_[index];
  3458. if (!data_op->HasAttr(ATTR_NAME_AIPP_INPUTS) || !data_op->HasAttr(ATTR_NAME_AIPP_OUTPUTS)) {
  3459. GELOGE(GE_AIPP_NOT_EXIST, "GetAllAippInputOutputDims: there is not AIPP related with index %u.", index);
  3460. return GE_AIPP_NOT_EXIST;
  3461. }
  3462. vector<std::string> inputs;
  3463. if (AttrUtils::GetListStr(data_op, ATTR_NAME_AIPP_INPUTS, inputs) && !inputs.empty()) {
  3464. GELOGI("GetAllAippInputOutputDims: Data: %s has %u related aippInfo.", data_op->GetName().c_str(), inputs.size());
  3465. for (auto it : inputs) {
  3466. InputOutputDims input_info;
  3467. ParseAIPPInfo(it, input_info);
  3468. input_dims.emplace_back(input_info);
  3469. GELOGD("GetAllAippInputOutputDims Aipp origin input dims info: %s", it.c_str());
  3470. ConstGeTensorDescPtr data_input_desc = data_op->GetInputDescPtr(kDataIndex);
  3471. int64_t data_input_size;
  3472. (void)TensorUtils::GetSize(*(data_op->GetInputDescPtr(kDataIndex)), data_input_size);
  3473. GELOGD(
  3474. "GetAllAippInputOutputDims related Data[%d]: tensor_name is %s, dim_num is %u, tensor_size: %zu, format: "
  3475. "%s, data_type: %s, shape: %s .",
  3476. index, data_op->GetName().c_str(), data_input_desc->GetShape().GetDimNum(), data_input_size,
  3477. TypeUtils::FormatToSerialString(data_input_desc->GetFormat()).c_str(),
  3478. TypeUtils::DataTypeToSerialString(data_input_desc->GetDataType()).c_str(),
  3479. formats::JoinToString(data_input_desc->GetShape().GetDims()).c_str());
  3480. }
  3481. }
  3482. vector<std::string> outputs;
  3483. if (AttrUtils::GetListStr(data_op, ATTR_NAME_AIPP_OUTPUTS, outputs) && !outputs.empty()) {
  3484. for (auto it : outputs) {
  3485. InputOutputDims output_info;
  3486. ParseAIPPInfo(it, output_info);
  3487. output_dims.emplace_back(output_info);
  3488. GELOGD("GetAllAippInputOutputDims Aipp output dims info: %s", it.c_str());
  3489. }
  3490. }
  3491. return SUCCESS;
  3492. }
  3493. int64_t DavinciModel::GetFixedAddrsSize(string tensor_name) {
  3494. if (tensor_name_to_fixed_addr_size_.find(tensor_name) != tensor_name_to_fixed_addr_size_.end()) {
  3495. return tensor_name_to_fixed_addr_size_[tensor_name];
  3496. } else {
  3497. return total_fixed_addr_size_;
  3498. }
  3499. }
  3500. void DavinciModel::GetFixedAddrAttr(const OpDescPtr &op_desc) {
  3501. if (op_desc->HasAttr(ATTR_DYNAMIC_SHAPE_FIXED_ADDR) && op_desc->HasAttr(ATTR_DYNAMIC_SHAPE_FIXED_ADDR_INDEX)) {
  3502. string tensor_name;
  3503. (void)AttrUtils::GetStr(op_desc, ATTR_DYNAMIC_SHAPE_FIXED_ADDR, tensor_name);
  3504. int64_t index = -1;
  3505. (void)AttrUtils::GetInt(op_desc, ATTR_DYNAMIC_SHAPE_FIXED_ADDR_INDEX, index);
  3506. if (index >= 0) {
  3507. tensor_name_to_peer_output_index_[tensor_name] = index;
  3508. }
  3509. }
  3510. }
  3511. } // namespace ge

图引擎模块(GE)是MindSpore的一个子模块,其代码由C++实现,位于前端模块ME和底层硬件之间,起到承接作用。图引擎模块以ME下发的图作为输入,然后进行一系列的深度图优化操作,最后输出一张可以在底层硬件上高效运行的图。GE针对昇腾AI处理器的硬件结构特点,做了特定的优化工作,以此来充分发挥出昇腾AI处理器的强大算力。在进行模型训练/推理时,GE会被自动调用而用户并不感知。GE主要由GE API和GE Core两部分组成,详细的架构图如下所示