You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 27 kB

3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
3 years ago
3 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
2 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983
  1. // Copyright 2016 The Gogs Authors. All rights reserved.
  2. // Copyright 2018 The Gitea Authors. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package repo
  6. import (
  7. "bufio"
  8. "encoding/json"
  9. "io"
  10. "net/http"
  11. "os"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "code.gitea.io/gitea/modules/notification"
  17. "code.gitea.io/gitea/modules/setting"
  18. "code.gitea.io/gitea/models"
  19. "code.gitea.io/gitea/modules/cloudbrain"
  20. "code.gitea.io/gitea/modules/context"
  21. "code.gitea.io/gitea/modules/log"
  22. "code.gitea.io/gitea/modules/modelarts"
  23. "code.gitea.io/gitea/modules/storage"
  24. routerRepo "code.gitea.io/gitea/routers/repo"
  25. )
  26. // cloudbrain get job task by jobid
  27. func GetCloudbrainTask(ctx *context.APIContext) {
  28. // swagger:operation GET /repos/{owner}/{repo}/cloudbrain/{jobid} cloudbrain jobTask
  29. // ---
  30. // summary: Get a single task
  31. // produces:
  32. // - application/json
  33. // parameters:
  34. // - name: owner
  35. // in: path
  36. // description: owner of the repo
  37. // type: string
  38. // required: true
  39. // - name: repo
  40. // in: path
  41. // description: name of the repo
  42. // type: string
  43. // required: true
  44. // - name: jobid
  45. // in: path
  46. // description: id of cloudbrain jobid
  47. // type: string
  48. // required: true
  49. // responses:
  50. // "200":
  51. // "$ref": "#/responses/Label"
  52. var (
  53. err error
  54. )
  55. ID := ctx.Params(":id")
  56. job, err := models.GetCloudbrainByID(ID)
  57. if err != nil {
  58. ctx.NotFound(err)
  59. log.Error("GetCloudbrainByID failed:", err)
  60. return
  61. }
  62. if job.JobType == string(models.JobTypeModelSafety) {
  63. routerRepo.GetAiSafetyTaskByJob(job)
  64. job, err = models.GetCloudbrainByID(ID)
  65. ctx.JSON(http.StatusOK, map[string]interface{}{
  66. "ID": ID,
  67. "JobName": job.JobName,
  68. "JobStatus": job.Status,
  69. "SubState": "",
  70. "CreatedTime": job.CreatedUnix.Format("2006-01-02 15:04:05"),
  71. "CompletedTime": job.UpdatedUnix.Format("2006-01-02 15:04:05"),
  72. "JobDuration": job.TrainJobDuration,
  73. })
  74. } else {
  75. jobResult, err := cloudbrain.GetJob(job.JobID)
  76. if err != nil {
  77. ctx.NotFound(err)
  78. log.Error("GetJob failed:", err)
  79. return
  80. }
  81. result, _ := models.ConvertToJobResultPayload(jobResult.Payload)
  82. if err != nil {
  83. ctx.NotFound(err)
  84. log.Error("ConvertToJobResultPayload failed:", err)
  85. return
  86. }
  87. oldStatus := job.Status
  88. job.Status = result.JobStatus.State
  89. taskRoles := result.TaskRoles
  90. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  91. if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
  92. job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  93. job.ContainerID = taskRes.TaskStatuses[0].ContainerID
  94. job.Status = taskRes.TaskStatuses[0].State
  95. }
  96. if result.JobStatus.State != string(models.JobWaiting) {
  97. models.ParseAndSetDurationFromCloudBrainOne(result, job)
  98. if oldStatus != job.Status {
  99. notification.NotifyChangeCloudbrainStatus(job, oldStatus)
  100. }
  101. err = models.UpdateJob(job)
  102. if err != nil {
  103. log.Error("UpdateJob failed:", err)
  104. }
  105. }
  106. ctx.JSON(http.StatusOK, map[string]interface{}{
  107. "ID": ID,
  108. "JobName": result.Config.JobName,
  109. "JobStatus": result.JobStatus.State,
  110. "SubState": result.JobStatus.SubState,
  111. "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"),
  112. "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"),
  113. "JobDuration": job.TrainJobDuration,
  114. })
  115. }
  116. }
  117. func GetCloudBrainInferenceJob(ctx *context.APIContext) {
  118. jobID := ctx.Params(":jobid")
  119. job, err := models.GetCloudbrainByJobID(jobID)
  120. if err != nil {
  121. ctx.NotFound(err)
  122. return
  123. }
  124. jobResult, err := cloudbrain.GetJob(job.JobID)
  125. if err != nil {
  126. ctx.NotFound(err)
  127. log.Error("GetJob failed:", err)
  128. return
  129. }
  130. result, err := models.ConvertToJobResultPayload(jobResult.Payload)
  131. if err != nil {
  132. ctx.NotFound(err)
  133. log.Error("ConvertToJobResultPayload failed:", err)
  134. return
  135. }
  136. oldStatus := job.Status
  137. job.Status = result.JobStatus.State
  138. if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
  139. taskRoles := result.TaskRoles
  140. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  141. job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  142. job.ContainerID = taskRes.TaskStatuses[0].ContainerID
  143. job.Status = taskRes.TaskStatuses[0].State
  144. }
  145. if result.JobStatus.State != string(models.JobWaiting) {
  146. models.ParseAndSetDurationFromCloudBrainOne(result, job)
  147. if oldStatus != job.Status {
  148. notification.NotifyChangeCloudbrainStatus(job, oldStatus)
  149. }
  150. err = models.UpdateJob(job)
  151. if err != nil {
  152. log.Error("UpdateJob failed:", err)
  153. }
  154. }
  155. ctx.JSON(http.StatusOK, map[string]interface{}{
  156. "JobID": jobID,
  157. "JobStatus": job.Status,
  158. "JobDuration": job.TrainJobDuration,
  159. "StartTime": job.StartTime,
  160. })
  161. }
  162. func DelCloudBrainJob(ctx *context.APIContext) {
  163. jobID := ctx.Params(":jobid")
  164. errStr := cloudbrain.DelCloudBrainJob(jobID)
  165. if errStr != "" {
  166. ctx.JSON(http.StatusOK, map[string]interface{}{
  167. "Message": ctx.Tr(errStr),
  168. "VersionName": "1",
  169. "Code": 1,
  170. })
  171. } else {
  172. ctx.JSON(http.StatusOK, map[string]interface{}{
  173. "Message": "",
  174. "VersionName": "1",
  175. "Code": 0,
  176. })
  177. }
  178. }
  179. func InferencJobResultList(ctx *context.APIContext) {
  180. jobID := ctx.Params(":jobid")
  181. parentDir := ctx.Query("parentDir")
  182. dirArray := strings.Split(parentDir, "/")
  183. task, err := models.GetCloudbrainByJobID(jobID)
  184. if err != nil {
  185. log.Error("get cloud brain err:", err)
  186. ctx.ServerError("get cloud brain information failed:", err)
  187. }
  188. //get dirs
  189. dirs, err := routerRepo.GetResultDirs(task.JobName, parentDir)
  190. if err != nil {
  191. log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  192. ctx.ServerError("GetModelDirs failed:", err)
  193. return
  194. }
  195. var fileInfos []storage.FileInfo
  196. err = json.Unmarshal([]byte(dirs), &fileInfos)
  197. if err != nil {
  198. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  199. ctx.ServerError("json.Unmarshal failed:", err)
  200. return
  201. }
  202. for i, fileInfo := range fileInfos {
  203. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  204. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  205. }
  206. sort.Slice(fileInfos, func(i, j int) bool {
  207. return fileInfos[i].ModTime > fileInfos[j].ModTime
  208. })
  209. ctx.JSON(http.StatusOK, map[string]interface{}{
  210. "JobID": jobID,
  211. "StatusOK": 0,
  212. "Path": dirArray,
  213. "Dirs": fileInfos,
  214. "task": task,
  215. "PageIsCloudBrain": true,
  216. })
  217. }
  218. func GetCloudbrainModelConvertTask(ctx *context.APIContext) {
  219. var (
  220. err error
  221. )
  222. ID := ctx.Params(":id")
  223. job, err := models.QueryModelConvertById(ID)
  224. if err != nil {
  225. ctx.NotFound(err)
  226. log.Error("GetCloudbrainByID failed:", err)
  227. return
  228. }
  229. if job.IsGpuTrainTask() {
  230. jobResult, err := cloudbrain.GetJob(job.CloudBrainTaskId)
  231. if err != nil {
  232. ctx.NotFound(err)
  233. log.Error("GetJob failed:", err)
  234. return
  235. }
  236. result, _ := models.ConvertToJobResultPayload(jobResult.Payload)
  237. if err != nil {
  238. ctx.NotFound(err)
  239. log.Error("ConvertToJobResultPayload failed:", err)
  240. return
  241. }
  242. job.Status = result.JobStatus.State
  243. taskRoles := result.TaskRoles
  244. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  245. if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
  246. job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  247. job.ContainerID = taskRes.TaskStatuses[0].ContainerID
  248. job.Status = taskRes.TaskStatuses[0].State
  249. }
  250. if result.JobStatus.State != string(models.JobWaiting) {
  251. models.ModelComputeAndSetDuration(job, result)
  252. err = models.UpdateModelConvert(job)
  253. if err != nil {
  254. log.Error("UpdateJob failed:", err)
  255. }
  256. }
  257. ctx.JSON(http.StatusOK, map[string]interface{}{
  258. "ID": ID,
  259. "JobName": result.Config.JobName,
  260. "JobStatus": result.JobStatus.State,
  261. "SubState": result.JobStatus.SubState,
  262. "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"),
  263. "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"),
  264. })
  265. } else {
  266. result, err := modelarts.GetTrainJob(job.CloudBrainTaskId, job.ModelArtsVersionId)
  267. if err != nil {
  268. log.Error("get modelart job failed:", err)
  269. ctx.NotFound(err)
  270. return
  271. }
  272. job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
  273. job.RunTime = result.Duration / 1000
  274. job.TrainJobDuration = models.ConvertDurationToStr(job.RunTime)
  275. err = models.UpdateModelConvert(job)
  276. if err != nil {
  277. log.Error("UpdateJob failed:", err)
  278. }
  279. ctx.JSON(http.StatusOK, map[string]interface{}{
  280. "ID": ID,
  281. "JobStatus": job.Status,
  282. })
  283. }
  284. }
  285. func CloudbrainGetLogByJobId(jobId string, jobName string) map[string]interface{} {
  286. var hits []models.Hits
  287. result, err := cloudbrain.GetJobLog(jobId)
  288. if err != nil {
  289. log.Error("GetJobLog failed: %v", err)
  290. return nil
  291. }
  292. hits = result.Hits.Hits
  293. //if the size equal page_size, then take the scroll_id to get all log and delete the scroll_id(the num of scroll_id is limited)
  294. if len(result.Hits.Hits) >= cloudbrain.LogPageSize {
  295. for {
  296. resultNext, err := cloudbrain.GetJobAllLog(result.ScrollID)
  297. if err != nil {
  298. log.Error("GetJobAllLog failed: %v", err)
  299. } else {
  300. for _, hit := range resultNext.Hits.Hits {
  301. hits = append(hits, hit)
  302. }
  303. }
  304. if len(resultNext.Hits.Hits) < cloudbrain.LogPageSize {
  305. log.Info("get all log already")
  306. break
  307. }
  308. }
  309. }
  310. cloudbrain.DeleteJobLogToken(result.ScrollID)
  311. sort.Slice(hits, func(i, j int) bool {
  312. return hits[i].Sort[0] < hits[j].Sort[0]
  313. })
  314. var content string
  315. for _, log := range hits {
  316. content += log.Source.Message + "\n"
  317. }
  318. return map[string]interface{}{
  319. "JobName": jobName,
  320. "Content": content,
  321. }
  322. }
  323. func CloudbrainForModelConvertGetLog(ctx *context.Context) {
  324. ID := ctx.Params(":id")
  325. job, err := models.QueryModelConvertById(ID)
  326. if err != nil {
  327. log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
  328. ctx.ServerError(err.Error(), err)
  329. return
  330. }
  331. result := CloudbrainGetLogByJobId(job.CloudBrainTaskId, job.Name)
  332. if result == nil {
  333. log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
  334. ctx.ServerError(err.Error(), err)
  335. return
  336. }
  337. ctx.JSON(http.StatusOK, result)
  338. }
  339. func ModelSafetyGetLog(ctx *context.APIContext) {
  340. ID := ctx.Params(":id")
  341. job, err := models.GetCloudbrainByID(ID)
  342. if err != nil {
  343. log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
  344. ctx.ServerError(err.Error(), err)
  345. return
  346. }
  347. if job.JobType == string(models.JobTypeModelSafety) {
  348. if job.Type == models.TypeCloudBrainTwo {
  349. //TrainJobForModelConvertGetLog(ctx)
  350. var baseLine = ctx.Query("base_line")
  351. var order = ctx.Query("order")
  352. var lines = ctx.Query("lines")
  353. lines_int, err := strconv.Atoi(lines)
  354. if err != nil {
  355. log.Error("change lines(%d) string to int failed", lines_int)
  356. }
  357. if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
  358. log.Error("order(%s) check failed", order)
  359. ctx.JSON(http.StatusBadRequest, map[string]interface{}{
  360. "err_msg": "order check failed",
  361. })
  362. return
  363. }
  364. resultLogFile, err := modelarts.GetTrainJobLogFileNames(job.JobID, strconv.FormatInt(job.VersionID, 10))
  365. if err != nil {
  366. log.Error("GetTrainJobLogFileNames(%s) failed:%v", job.JobID, err.Error())
  367. }
  368. result, err := modelarts.GetTrainJobLog(job.JobID, strconv.FormatInt(job.VersionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines_int)
  369. if err != nil {
  370. log.Error("GetTrainJobLog(%s) failed:%v", job.JobID, err.Error())
  371. }
  372. if err != nil {
  373. log.Error("trainJobGetLog(%s) failed:%v", job.JobID, err.Error())
  374. // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
  375. ctx.JSON(http.StatusOK, map[string]interface{}{
  376. "JobID": job.JobID,
  377. "LogFileName": "",
  378. "StartLine": "0",
  379. "EndLine": "0",
  380. "Content": "",
  381. "Lines": 0,
  382. "CanLogDownload": false,
  383. })
  384. return
  385. }
  386. ctx.Data["log_file_name"] = resultLogFile.LogFileList[0]
  387. ctx.JSON(http.StatusOK, map[string]interface{}{
  388. "JobID": job.JobID,
  389. "LogFileName": resultLogFile.LogFileList[0],
  390. "StartLine": result.StartLine,
  391. "EndLine": result.EndLine,
  392. "Content": result.Content,
  393. "Lines": result.Lines,
  394. "CanLogDownload": isCanDownloadLog(ctx, job),
  395. "StartTime": job.StartTime,
  396. })
  397. }
  398. }
  399. //result := ""
  400. //ctx.JSON(http.StatusOK, result)
  401. }
  402. func isCanDownloadLog(ctx *context.APIContext, job *models.Cloudbrain) bool {
  403. if !ctx.IsSigned {
  404. return false
  405. }
  406. return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID
  407. }
  408. func ModelSafetyDownloadLogFile(ctx *context.Context) {
  409. ID := ctx.Params(":id")
  410. job, err := models.GetCloudbrainByID(ID)
  411. if err != nil {
  412. log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
  413. ctx.ServerError(err.Error(), err)
  414. return
  415. }
  416. if job.JobType == string(models.JobTypeModelSafety) {
  417. if job.Type == models.TypeCloudBrainOne {
  418. CloudbrainDownloadLogFile(ctx)
  419. } else if job.Type == models.TypeCloudBrainTwo {
  420. ctx.SetParams("jobid", job.JobID)
  421. ctx.Req.Form.Set("version_name", job.VersionName)
  422. routerRepo.TrainJobDownloadLogFile(ctx)
  423. }
  424. }
  425. }
  426. func CloudbrainDownloadLogFile(ctx *context.Context) {
  427. ID := ctx.Params(":id")
  428. job, err := models.GetCloudbrainByID(ID)
  429. if err != nil {
  430. log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
  431. ctx.ServerError(err.Error(), err)
  432. return
  433. }
  434. if job.JobType == string(models.JobTypeModelSafety) {
  435. if job.Type == models.TypeCloudBrainTwo {
  436. ModelSafetyDownloadLogFile(ctx)
  437. return
  438. }
  439. }
  440. logDir := "/model"
  441. if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) {
  442. logDir = cloudbrain.ResultPath
  443. }
  444. files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix+job.JobName+logDir, "")
  445. if err != nil {
  446. log.Error("query cloudbrain model failed: %v", err)
  447. return
  448. }
  449. fileName := ""
  450. for _, file := range files {
  451. if strings.HasSuffix(file.FileName, "log.txt") {
  452. fileName = file.FileName
  453. break
  454. }
  455. }
  456. if fileName != "" {
  457. prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir
  458. url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName)
  459. if err != nil {
  460. log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
  461. ctx.ServerError("Get minio get SignedUrl failed", err)
  462. return
  463. }
  464. log.Info("fileName=" + fileName)
  465. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
  466. } else {
  467. log.Info("fileName is null.")
  468. }
  469. }
  470. func CloudbrainGetLog(ctx *context.APIContext) {
  471. ID := ctx.Params(":id")
  472. job, err := models.GetCloudbrainByID(ID)
  473. if err != nil {
  474. log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
  475. ctx.ServerError(err.Error(), err)
  476. return
  477. }
  478. if job.JobType == string(models.JobTypeModelSafety) {
  479. if job.Type == models.TypeCloudBrainOne {
  480. result, err := cloudbrain.GetJob(job.JobID)
  481. existStr := ""
  482. if err == nil && result != nil {
  483. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  484. taskRoles := jobRes.TaskRoles
  485. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  486. existStr = taskRes.TaskStatuses[0].ExitDiagnostics
  487. }
  488. ctx.Data["existStr"] = existStr
  489. log.Info("existStr=" + existStr)
  490. } else {
  491. ModelSafetyGetLog(ctx)
  492. return
  493. }
  494. }
  495. lines := ctx.QueryInt("lines")
  496. baseLine := ctx.Query("base_line")
  497. order := ctx.Query("order")
  498. var result map[string]interface{}
  499. resultPath := "/model"
  500. if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) {
  501. resultPath = "/result"
  502. }
  503. if baseLine == "" && order == "desc" {
  504. result = getLastLogFromModelDir(job.JobName, lines, resultPath)
  505. } else {
  506. startLine := ctx.QueryInt("base_line")
  507. endLine := startLine + lines
  508. if order == "asc" {
  509. if baseLine == "" {
  510. startLine = 0
  511. endLine = lines
  512. } else {
  513. endLine = startLine
  514. startLine = endLine - lines
  515. if startLine < 0 {
  516. startLine = 0
  517. }
  518. }
  519. } else {
  520. if startLine > 0 {
  521. startLine += 1
  522. endLine += 1
  523. }
  524. }
  525. result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
  526. if result == nil {
  527. log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
  528. ctx.ServerError(err.Error(), err)
  529. return
  530. }
  531. }
  532. content := ""
  533. if result["Content"] != nil {
  534. content = result["Content"].(string)
  535. }
  536. if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
  537. content = content + ctx.Data["existStr"].(string)
  538. }
  539. re := map[string]interface{}{
  540. "JobID": ID,
  541. "LogFileName": result["FileName"],
  542. "StartLine": result["StartLine"],
  543. "EndLine": result["EndLine"],
  544. "Content": content,
  545. "Lines": result["Lines"],
  546. "CanLogDownload": result["FileName"] != "",
  547. "StartTime": job.StartTime,
  548. }
  549. //result := CloudbrainGetLogByJobId(job.JobID, job.JobName)
  550. ctx.JSON(http.StatusOK, re)
  551. }
  552. func getAllLineFromFile(path string) int {
  553. count := 0
  554. reader, err := os.Open(path)
  555. defer reader.Close()
  556. if err == nil {
  557. r := bufio.NewReader(reader)
  558. for {
  559. _, error := r.ReadString('\n')
  560. if error == io.EOF {
  561. log.Info("read file completed.")
  562. break
  563. }
  564. if error != nil {
  565. log.Info("read file error." + error.Error())
  566. break
  567. }
  568. count = count + 1
  569. }
  570. } else {
  571. log.Info("error:" + err.Error())
  572. }
  573. return count
  574. }
  575. func getLastLogFromModelDir(jobName string, lines int, resultPath string) map[string]interface{} {
  576. prefix := setting.CBCodePathPrefix + jobName + resultPath
  577. files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
  578. if err != nil {
  579. log.Error("query cloudbrain model failed: %v", err)
  580. return nil
  581. }
  582. re := ""
  583. fileName := ""
  584. count := 0
  585. allLines := 0
  586. startLine := 0
  587. for _, file := range files {
  588. if strings.HasSuffix(file.FileName, "log.txt") {
  589. fileName = file.FileName
  590. path := storage.GetMinioPath(jobName+resultPath+"/", file.FileName)
  591. allLines = getAllLineFromFile(path)
  592. startLine = allLines - lines
  593. if startLine < 0 {
  594. startLine = 0
  595. }
  596. count = allLines - startLine
  597. log.Info("path=" + path)
  598. reader, err := os.Open(path)
  599. defer reader.Close()
  600. if err == nil {
  601. r := bufio.NewReader(reader)
  602. for i := 0; i < allLines; i++ {
  603. line, error := r.ReadString('\n')
  604. if error == io.EOF {
  605. log.Info("read file completed.")
  606. break
  607. }
  608. if error != nil {
  609. log.Info("read file error." + error.Error())
  610. break
  611. }
  612. if error == nil {
  613. if i >= startLine {
  614. re = re + line
  615. }
  616. }
  617. }
  618. } else {
  619. log.Info("error:" + err.Error())
  620. }
  621. break
  622. }
  623. }
  624. return map[string]interface{}{
  625. "JobName": jobName,
  626. "Content": re,
  627. "FileName": fileName,
  628. "Lines": count,
  629. "EndLine": allLines,
  630. "StartLine": startLine,
  631. }
  632. }
  633. func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath string) map[string]interface{} {
  634. prefix := setting.CBCodePathPrefix + jobName + resultPath
  635. files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
  636. if err != nil {
  637. log.Error("query cloudbrain model failed: %v", err)
  638. return nil
  639. }
  640. if startLine == endLine {
  641. return map[string]interface{}{
  642. "JobName": jobName,
  643. "Content": "",
  644. "FileName": "",
  645. "Lines": 0,
  646. "EndLine": startLine,
  647. "StartLine": startLine,
  648. }
  649. }
  650. re := ""
  651. fileName := ""
  652. count := 0
  653. fileEndLine := endLine
  654. for _, file := range files {
  655. if strings.HasSuffix(file.FileName, "log.txt") {
  656. fileName = file.FileName
  657. path := storage.GetMinioPath(jobName+resultPath+"/", file.FileName)
  658. log.Info("path=" + path)
  659. reader, err := os.Open(path)
  660. defer reader.Close()
  661. if err == nil {
  662. r := bufio.NewReader(reader)
  663. for i := 0; i < endLine; i++ {
  664. line, error := r.ReadString('\n')
  665. if error == io.EOF {
  666. if i >= startLine {
  667. re = re + line
  668. count++
  669. }
  670. fileEndLine = i + 1
  671. log.Info("read file completed.")
  672. break
  673. }
  674. if error != nil {
  675. log.Info("read file error." + error.Error())
  676. break
  677. }
  678. if error == nil {
  679. if i >= startLine {
  680. fileEndLine = i + 1
  681. re = re + line
  682. count++
  683. }
  684. }
  685. }
  686. } else {
  687. log.Info("error:" + err.Error())
  688. }
  689. break
  690. }
  691. }
  692. return map[string]interface{}{
  693. "JobName": jobName,
  694. "Content": re,
  695. "FileName": fileName,
  696. "Lines": count,
  697. "EndLine": fileEndLine,
  698. "StartLine": startLine,
  699. }
  700. }
  701. func CloudBrainModelConvertList(ctx *context.APIContext) {
  702. var (
  703. err error
  704. )
  705. ID := ctx.Params(":id")
  706. parentDir := ctx.Query("parentDir")
  707. dirArray := strings.Split(parentDir, "/")
  708. job, err := models.QueryModelConvertById(ID)
  709. if err != nil {
  710. log.Error("GetCloudbrainByJobID(%s) failed:%v", job.Name, err.Error())
  711. return
  712. }
  713. if job.IsGpuTrainTask() {
  714. //get dirs
  715. dirs, err := routerRepo.GetModelDirs(job.ID, parentDir)
  716. if err != nil {
  717. log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  718. ctx.ServerError("GetModelDirs failed:", err)
  719. return
  720. }
  721. var fileInfos []storage.FileInfo
  722. err = json.Unmarshal([]byte(dirs), &fileInfos)
  723. if err != nil {
  724. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  725. ctx.ServerError("json.Unmarshal failed:", err)
  726. return
  727. }
  728. for i, fileInfo := range fileInfos {
  729. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  730. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  731. }
  732. sort.Slice(fileInfos, func(i, j int) bool {
  733. return fileInfos[i].ModTime > fileInfos[j].ModTime
  734. })
  735. ctx.JSON(http.StatusOK, map[string]interface{}{
  736. "JobID": ID,
  737. "VersionName": "",
  738. "StatusOK": 0,
  739. "Path": dirArray,
  740. "Dirs": fileInfos,
  741. "task": job,
  742. "PageIsCloudBrain": true,
  743. })
  744. } else {
  745. var jobID = ctx.Params(":id")
  746. var versionName = "V0001"
  747. parentDir := ctx.Query("parentDir")
  748. dirArray := strings.Split(parentDir, "/")
  749. models, err := storage.GetObsListObject(job.ID, "output/", parentDir, versionName)
  750. if err != nil {
  751. log.Info("get TrainJobListModel failed:", err)
  752. ctx.ServerError("GetObsListObject:", err)
  753. return
  754. }
  755. ctx.JSON(http.StatusOK, map[string]interface{}{
  756. "JobID": jobID,
  757. "VersionName": versionName,
  758. "StatusOK": 0,
  759. "Path": dirArray,
  760. "Dirs": models,
  761. "task": job,
  762. "PageIsCloudBrain": true,
  763. })
  764. }
  765. }
  766. func CloudBrainModelList(ctx *context.APIContext) {
  767. var (
  768. err error
  769. )
  770. var jobID = ctx.Params(":jobid")
  771. var versionName = ctx.Query("version_name")
  772. parentDir := ctx.Query("parentDir")
  773. dirArray := strings.Split(parentDir, "/")
  774. task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
  775. if err != nil {
  776. log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
  777. return
  778. }
  779. //get dirs
  780. dirs, err := routerRepo.GetModelDirs(task.JobName, parentDir)
  781. if err != nil {
  782. log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  783. ctx.ServerError("GetModelDirs failed:", err)
  784. return
  785. }
  786. var fileInfos []storage.FileInfo
  787. err = json.Unmarshal([]byte(dirs), &fileInfos)
  788. if err != nil {
  789. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  790. ctx.ServerError("json.Unmarshal failed:", err)
  791. return
  792. }
  793. for i, fileInfo := range fileInfos {
  794. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  795. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  796. }
  797. sort.Slice(fileInfos, func(i, j int) bool {
  798. return fileInfos[i].ModTime > fileInfos[j].ModTime
  799. })
  800. ctx.JSON(http.StatusOK, map[string]interface{}{
  801. "JobID": jobID,
  802. "VersionName": versionName,
  803. "StatusOK": 0,
  804. "Path": dirArray,
  805. "Dirs": fileInfos,
  806. "task": task,
  807. "PageIsCloudBrain": true,
  808. })
  809. }
  810. type JobInfo struct {
  811. JobName string `json:"job_name"`
  812. AiCenterId int `json:"ai_center_id"`
  813. }
  814. func GetNewestJobs(ctx *context.APIContext) {
  815. idsC2Net, err := models.GetNewestJobsByAiCenter()
  816. if err != nil {
  817. log.Error("GetNewestJobsByAiCenter(%s) failed:%v", err.Error())
  818. return
  819. }
  820. idsCloudbrain, err := models.GetNewestJobsByType()
  821. if err != nil {
  822. log.Error("GetNewestJobsByType(%s) failed:%v", err.Error())
  823. return
  824. }
  825. ids := make([]int64, len(idsC2Net), cap(idsC2Net)*2)
  826. copy(ids, idsC2Net)
  827. for _, id := range idsCloudbrain {
  828. ids = append(ids, id)
  829. }
  830. jobs, err := models.GetCloudbrainByIDs(ids)
  831. if err != nil {
  832. log.Error("GetCloudbrainByIDs(%s) failed:%v", err.Error())
  833. return
  834. }
  835. jobInfos := make([]JobInfo, 0)
  836. for _, job := range jobs {
  837. var id int
  838. var content string
  839. switch job.Type {
  840. case models.TypeCloudBrainOne:
  841. id, content = getAICenterID("cloudbrain_one")
  842. if content == "" {
  843. log.Error("job(%s) has no match config info", job.DisplayJobName)
  844. continue
  845. }
  846. case models.TypeCloudBrainTwo:
  847. id, content = getAICenterID("cloudbrain_two")
  848. if content == "" {
  849. log.Error("job(%s) has no match config info", job.DisplayJobName)
  850. continue
  851. }
  852. case models.TypeC2Net:
  853. centerInfo := strings.Split(job.AiCenter, "+")
  854. if len(centerInfo) != 2 {
  855. log.Error("job(%s):ai_center(%s) is wrong", job.DisplayJobName, job.AiCenter)
  856. continue
  857. }
  858. id, content = getAICenterID(centerInfo[0])
  859. if content == "" {
  860. log.Error("job(%s) has no match config info", job.DisplayJobName)
  861. continue
  862. }
  863. default:
  864. log.Error("no match info")
  865. continue
  866. }
  867. jobInfos = append(jobInfos, JobInfo{
  868. JobName: job.DisplayJobName,
  869. AiCenterId: id,
  870. })
  871. }
  872. ctx.JSON(http.StatusOK, jobInfos)
  873. }
  874. func GetAICenterInfo(ctx *context.APIContext) {
  875. if setting.C2NetInfos == nil {
  876. log.Error("C2NET_SEQUENCE is incorrect")
  877. return
  878. }
  879. ctx.JSON(http.StatusOK, setting.C2NetInfos.C2NetSqInfo)
  880. }
  881. func getAICenterID(name string) (int, string) {
  882. for _, info := range setting.C2NetInfos.C2NetSqInfo {
  883. if name == info.Name {
  884. return info.ID, info.Content
  885. }
  886. }
  887. return 0, ""
  888. }