You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

cloudbrain.go 20 kB

4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
4 years ago
5 years ago
5 years ago
5 years ago
3 years ago
3 years ago
4 years ago
3 years ago
5 years ago
4 years ago
5 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718
  1. package repo
  2. import (
  3. "bufio"
  4. "encoding/json"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "net/http"
  9. "os"
  10. "os/exec"
  11. "regexp"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "code.gitea.io/gitea/modules/modelarts"
  17. "code.gitea.io/gitea/modules/git"
  18. "code.gitea.io/gitea/modules/storage"
  19. "code.gitea.io/gitea/models"
  20. "code.gitea.io/gitea/modules/auth"
  21. "code.gitea.io/gitea/modules/base"
  22. "code.gitea.io/gitea/modules/cloudbrain"
  23. "code.gitea.io/gitea/modules/context"
  24. "code.gitea.io/gitea/modules/log"
  25. "code.gitea.io/gitea/modules/setting"
  26. )
  27. const (
  28. tplCloudBrainIndex base.TplName = "repo/cloudbrain/index"
  29. tplCloudBrainNew base.TplName = "repo/cloudbrain/new"
  30. tplCloudBrainShow base.TplName = "repo/cloudbrain/show"
  31. tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"
  32. )
  33. var (
  34. gpuInfos *models.GpuInfos
  35. categories *models.Categories
  36. )
  37. var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
  38. // MustEnableDataset check if repository enable internal cb
  39. func MustEnableCloudbrain(ctx *context.Context) {
  40. if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
  41. ctx.NotFound("MustEnableCloudbrain", nil)
  42. return
  43. }
  44. }
  45. func CloudBrainIndex(ctx *context.Context) {
  46. MustEnableCloudbrain(ctx)
  47. repo := ctx.Repo.Repository
  48. page := ctx.QueryInt("page")
  49. if page <= 0 {
  50. page = 1
  51. }
  52. ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
  53. ListOptions: models.ListOptions{
  54. Page: page,
  55. PageSize: setting.UI.IssuePagingNum,
  56. },
  57. RepoID: repo.ID,
  58. Type: models.TypeCloudBrainOne,
  59. })
  60. if err != nil {
  61. ctx.ServerError("Cloudbrain", err)
  62. return
  63. }
  64. timestamp := time.Now().Unix()
  65. for i, task := range ciTasks {
  66. if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) {
  67. ciTasks[i].CanDebug = true
  68. } else {
  69. ciTasks[i].CanDebug = false
  70. }
  71. ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task)
  72. }
  73. pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
  74. pager.SetDefaultParams(ctx)
  75. ctx.Data["Page"] = pager
  76. ctx.Data["PageIsCloudBrain"] = true
  77. ctx.Data["Tasks"] = ciTasks
  78. ctx.HTML(200, tplCloudBrainIndex)
  79. }
  80. func cutString(str string, lens int) string {
  81. if len(str) < lens {
  82. return str
  83. }
  84. return str[:lens]
  85. }
  86. func jobNamePrefixValid(s string) string {
  87. lowStr := strings.ToLower(s)
  88. re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
  89. removeSpecial := re.ReplaceAllString(lowStr, "")
  90. re = regexp.MustCompile(`^[_\\-]+`)
  91. return re.ReplaceAllString(removeSpecial, "")
  92. }
  93. func cloudBrainNewDataPrepare(ctx *context.Context) error {
  94. ctx.Data["PageIsCloudBrain"] = true
  95. t := time.Now()
  96. var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
  97. ctx.Data["job_name"] = jobName
  98. result, err := cloudbrain.GetImages()
  99. if err != nil {
  100. ctx.Data["error"] = err.Error()
  101. log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"])
  102. }
  103. for i, payload := range result.Payload.ImageInfo {
  104. if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") {
  105. result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  106. } else {
  107. result.Payload.ImageInfo[i].PlaceView = payload.Place
  108. }
  109. }
  110. ctx.Data["images"] = result.Payload.ImageInfo
  111. resultPublic, err := cloudbrain.GetPublicImages()
  112. if err != nil {
  113. ctx.Data["error"] = err.Error()
  114. log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"])
  115. }
  116. for i, payload := range resultPublic.Payload.ImageInfo {
  117. if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") {
  118. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)]
  119. } else {
  120. resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place
  121. }
  122. }
  123. ctx.Data["public_images"] = resultPublic.Payload.ImageInfo
  124. attachs, err := models.GetAllUserAttachments(ctx.User.ID)
  125. if err != nil {
  126. log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"])
  127. return err
  128. }
  129. ctx.Data["attachments"] = attachs
  130. ctx.Data["command"] = cloudbrain.Command
  131. ctx.Data["code_path"] = cloudbrain.CodeMountPath
  132. ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
  133. ctx.Data["model_path"] = cloudbrain.ModelMountPath
  134. ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
  135. ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
  136. if categories == nil {
  137. json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
  138. }
  139. ctx.Data["benchmark_categories"] = categories.Category
  140. if gpuInfos == nil {
  141. json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos)
  142. }
  143. ctx.Data["gpu_types"] = gpuInfos.GpuInfo
  144. if cloudbrain.ResourceSpecs == nil {
  145. json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
  146. }
  147. ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec
  148. ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
  149. ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled
  150. ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
  151. ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled
  152. return nil
  153. }
  154. func CloudBrainNew(ctx *context.Context) {
  155. err := cloudBrainNewDataPrepare(ctx)
  156. if err != nil {
  157. ctx.ServerError("get new cloudbrain info failed", err)
  158. return
  159. }
  160. ctx.HTML(200, tplCloudBrainNew)
  161. }
  162. func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
  163. ctx.Data["PageIsCloudBrain"] = true
  164. jobName := form.JobName
  165. image := form.Image
  166. command := form.Command
  167. uuid := form.Attachment
  168. jobType := form.JobType
  169. gpuQueue := setting.JobType
  170. codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
  171. resourceSpecId := form.ResourceSpecId
  172. if !jobNamePattern.MatchString(jobName) {
  173. ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainNew, &form)
  174. return
  175. }
  176. if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
  177. log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
  178. cloudBrainNewDataPrepare(ctx)
  179. ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form)
  180. return
  181. }
  182. _, err := models.GetCloudbrainByName(jobName)
  183. if err == nil {
  184. log.Error("the job name did already exist", ctx.Data["MsgID"])
  185. cloudBrainNewDataPrepare(ctx)
  186. ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form)
  187. return
  188. } else {
  189. if !models.IsErrJobNotExist(err) {
  190. log.Error("system error, %v", err, ctx.Data["MsgID"])
  191. cloudBrainNewDataPrepare(ctx)
  192. ctx.RenderWithErr("system error", tplCloudBrainNew, &form)
  193. return
  194. }
  195. }
  196. repo := ctx.Repo.Repository
  197. downloadCode(repo, codePath)
  198. modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
  199. err = os.MkdirAll(modelPath, os.ModePerm)
  200. if err != nil {
  201. cloudBrainNewDataPrepare(ctx)
  202. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  203. return
  204. }
  205. benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
  206. if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
  207. gpuQueue = form.GpuType
  208. var gpuType string
  209. for _, gpuInfo := range gpuInfos.GpuInfo {
  210. if gpuInfo.Queue == gpuQueue {
  211. gpuType = gpuInfo.Value
  212. }
  213. }
  214. downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
  215. }
  216. snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
  217. if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
  218. downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
  219. }
  220. brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
  221. if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
  222. downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "")
  223. }
  224. err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId)
  225. if err != nil {
  226. cloudBrainNewDataPrepare(ctx)
  227. ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
  228. return
  229. }
  230. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  231. }
  232. func CloudBrainShow(ctx *context.Context) {
  233. ctx.Data["PageIsCloudBrain"] = true
  234. var jobID = ctx.Params(":jobid")
  235. task, err := models.GetCloudbrainByJobID(jobID)
  236. if err != nil {
  237. ctx.Data["error"] = err.Error()
  238. }
  239. result, err := cloudbrain.GetJob(jobID)
  240. if err != nil {
  241. ctx.Data["error"] = err.Error()
  242. }
  243. if result != nil {
  244. jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
  245. jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
  246. taskRoles := jobRes.TaskRoles
  247. if jobRes.JobStatus.State != string(models.JobFailed) {
  248. taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
  249. ctx.Data["taskRes"] = taskRes
  250. task.Status = taskRes.TaskStatuses[0].State
  251. task.ContainerID = taskRes.TaskStatuses[0].ContainerID
  252. task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
  253. err = models.UpdateJob(task)
  254. if err != nil {
  255. ctx.Data["error"] = err.Error()
  256. }
  257. } else {
  258. task.Status = jobRes.JobStatus.State
  259. taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
  260. {
  261. State: jobRes.JobStatus.State,
  262. },
  263. }}
  264. ctx.Data["taskRes"] = taskRes
  265. jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
  266. jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
  267. }
  268. ctx.Data["result"] = jobRes
  269. }
  270. ctx.Data["task"] = task
  271. ctx.Data["jobID"] = jobID
  272. ctx.HTML(200, tplCloudBrainShow)
  273. }
  274. func CloudBrainDebug(ctx *context.Context) {
  275. var jobID = ctx.Params(":jobid")
  276. if !ctx.IsSigned {
  277. log.Error("the user has not signed in")
  278. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  279. return
  280. }
  281. task, err := models.GetCloudbrainByJobID(jobID)
  282. if err != nil {
  283. ctx.ServerError("GetCloudbrainByJobID failed", err)
  284. return
  285. }
  286. debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
  287. ctx.Redirect(debugUrl)
  288. }
  289. func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
  290. var jobID = ctx.Params(":jobid")
  291. if !ctx.IsSigned {
  292. log.Error("the user has not signed in")
  293. ctx.Error(http.StatusForbidden, "","the user has not signed in")
  294. return
  295. }
  296. task, err := models.GetCloudbrainByJobID(jobID)
  297. if err != nil {
  298. ctx.JSON(200, map[string]string{
  299. "result_code": "-1",
  300. "error_msg": "GetCloudbrainByJobID failed",
  301. })
  302. return
  303. }
  304. err = cloudbrain.CommitImage(jobID, models.CommitImageParams{
  305. Ip: task.ContainerIp,
  306. TaskContainerId: task.ContainerID,
  307. ImageDescription: form.Description,
  308. ImageTag: form.Tag,
  309. })
  310. if err != nil {
  311. log.Error("CommitImage(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  312. ctx.JSON(200, map[string]string{
  313. "result_code": "-1",
  314. "error_msg": "CommitImage failed",
  315. })
  316. return
  317. }
  318. ctx.JSON(200, map[string]string{
  319. "result_code": "0",
  320. "error_msg": "",
  321. })
  322. }
  323. func CloudBrainStop(ctx *context.Context) {
  324. var jobID = ctx.Params(":jobid")
  325. task, err := models.GetCloudbrainByJobID(jobID)
  326. if err != nil {
  327. ctx.ServerError("GetCloudbrainByJobID failed", err)
  328. return
  329. }
  330. if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) {
  331. log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
  332. ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
  333. return
  334. }
  335. err = cloudbrain.StopJob(jobID)
  336. if err != nil {
  337. log.Error("StopJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"])
  338. ctx.ServerError("StopJob failed", err)
  339. return
  340. }
  341. task.Status = string(models.JobStopped)
  342. err = models.UpdateJob(task)
  343. if err != nil {
  344. ctx.ServerError("UpdateJob failed", err)
  345. return
  346. }
  347. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  348. }
  349. func StopJobsByUserID(userID int64) {
  350. cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
  351. if err != nil {
  352. log.Warn("Failed to get cloudBrain info", err)
  353. return
  354. }
  355. StopJobs(cloudBrains)
  356. }
  357. func StopJobsByRepoID(repoID int64) {
  358. cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
  359. if err != nil {
  360. log.Warn("Failed to get cloudBrain info", err)
  361. return
  362. }
  363. StopJobs(cloudBrains)
  364. }
  365. /**
  366. */
  367. func StopJobs(cloudBrains []*models.Cloudbrain) {
  368. for _, taskInfo := range cloudBrains {
  369. if taskInfo.Type == models.TypeCloudBrainOne {
  370. err := retry(3, time.Second*30, func() error {
  371. return cloudbrain.StopJob(taskInfo.JobID)
  372. })
  373. logErrorAndUpdateJobStatus(err, taskInfo)
  374. } else {
  375. param := models.NotebookAction{
  376. Action: models.ActionStop,
  377. }
  378. err := retry(3, time.Second*30, func() error {
  379. _, err := modelarts.StopJob(taskInfo.JobID, param)
  380. return err
  381. })
  382. logErrorAndUpdateJobStatus(err, taskInfo)
  383. }
  384. }
  385. }
  386. func retry(attempts int, sleep time.Duration, f func() error) (err error) {
  387. for i := 0; i < attempts; i++ {
  388. if i > 0 {
  389. log.Warn("retrying after error:", err)
  390. time.Sleep(sleep)
  391. }
  392. err = f()
  393. if err == nil {
  394. return nil
  395. }
  396. }
  397. return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
  398. }
  399. func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
  400. if err != nil {
  401. log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
  402. } else {
  403. taskInfo.Status = string(models.JobStopped)
  404. err = models.UpdateJob(taskInfo)
  405. if err != nil {
  406. log.Warn("UpdateJob failed", err)
  407. }
  408. }
  409. }
  410. func CloudBrainDel(ctx *context.Context) {
  411. var jobID = ctx.Params(":jobid")
  412. task, err := models.GetCloudbrainByJobID(jobID)
  413. if err != nil {
  414. ctx.ServerError("GetCloudbrainByJobID failed", err)
  415. return
  416. }
  417. if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) {
  418. log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
  419. ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
  420. return
  421. }
  422. err = models.DeleteJob(task)
  423. if err != nil {
  424. ctx.ServerError("DeleteJob failed", err)
  425. return
  426. }
  427. ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
  428. }
  429. func CloudBrainShowModels(ctx *context.Context) {
  430. ctx.Data["PageIsCloudBrain"] = true
  431. jobID := ctx.Params(":jobid")
  432. parentDir := ctx.Query("parentDir")
  433. dirArray := strings.Split(parentDir, "/")
  434. task, err := models.GetCloudbrainByJobID(jobID)
  435. if err != nil {
  436. log.Error("no such job!", ctx.Data["msgID"])
  437. ctx.ServerError("no such job:", err)
  438. return
  439. }
  440. //get dirs
  441. dirs, err := getModelDirs(task.JobName, parentDir)
  442. if err != nil {
  443. log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
  444. ctx.ServerError("getModelDirs failed:", err)
  445. return
  446. }
  447. var fileInfos []storage.FileInfo
  448. err = json.Unmarshal([]byte(dirs), &fileInfos)
  449. if err != nil {
  450. log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
  451. ctx.ServerError("json.Unmarshal failed:", err)
  452. return
  453. }
  454. for i, fileInfo := range fileInfos {
  455. temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
  456. fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
  457. }
  458. sort.Slice(fileInfos, func(i, j int) bool {
  459. return fileInfos[i].ModTime > fileInfos[j].ModTime
  460. })
  461. ctx.Data["Path"] = dirArray
  462. ctx.Data["Dirs"] = fileInfos
  463. ctx.Data["task"] = task
  464. ctx.Data["JobID"] = jobID
  465. ctx.HTML(200, tplCloudBrainShowModels)
  466. }
  467. func GetPublicImages(ctx *context.Context) {
  468. getImages(ctx, cloudbrain.Public)
  469. }
  470. func GetCustomImages(ctx *context.Context) {
  471. getImages(ctx, cloudbrain.Custom)
  472. }
  473. func getImages(ctx *context.Context, imageType string) {
  474. log.Info("Get images begin")
  475. page := ctx.QueryInt("page")
  476. size := ctx.QueryInt("size")
  477. name := ctx.Query("name")
  478. getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name)
  479. if err != nil {
  480. log.Error("Can not get images:%v", err)
  481. ctx.JSON(http.StatusOK, models.GetImagesPayload{
  482. Count: 0,
  483. TotalPages: 0,
  484. ImageInfo: []*models.ImageInfo{},
  485. })
  486. } else {
  487. ctx.JSON(http.StatusOK, getImagesResult.Payload)
  488. }
  489. log.Info("Get images end")
  490. }
  491. func getModelDirs(jobName string, parentDir string) (string, error) {
  492. var req string
  493. modelActualPath := setting.JobPath + jobName + "/model/"
  494. if parentDir == "" {
  495. req = "baseDir=" + modelActualPath
  496. } else {
  497. req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
  498. }
  499. return getDirs(req)
  500. }
  501. func CloudBrainDownloadModel(ctx *context.Context) {
  502. parentDir := ctx.Query("parentDir")
  503. fileName := ctx.Query("fileName")
  504. jobName := ctx.Query("jobName")
  505. filePath := "jobs/" + jobName + "/model/" + parentDir
  506. url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
  507. if err != nil {
  508. log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
  509. ctx.ServerError("PresignedGetURL", err)
  510. return
  511. }
  512. http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
  513. }
  514. func GetRate(ctx *context.Context) {
  515. var jobID = ctx.Params(":jobid")
  516. job, err := models.GetCloudbrainByJobID(jobID)
  517. if err != nil {
  518. ctx.ServerError("GetCloudbrainByJobID failed", err)
  519. return
  520. }
  521. if job.JobType == string(models.JobTypeBenchmark) {
  522. ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
  523. } else if job.JobType == string(models.JobTypeSnn4imagenet) {
  524. ctx.Redirect(setting.Snn4imagenetServerHost)
  525. } else if job.JobType == string(models.JobTypeBrainScore) {
  526. ctx.Redirect(setting.BrainScoreServerHost)
  527. } else {
  528. log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
  529. }
  530. }
  531. func downloadCode(repo *models.Repository, codePath string) error {
  532. if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
  533. log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
  534. return err
  535. }
  536. configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
  537. if err != nil {
  538. log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
  539. return err
  540. }
  541. defer configFile.Close()
  542. pos := int64(0)
  543. reader := bufio.NewReader(configFile)
  544. for {
  545. line, err := reader.ReadString('\n')
  546. if err != nil {
  547. if err == io.EOF {
  548. log.Error("not find the remote-url")
  549. return nil
  550. } else {
  551. log.Error("read error: %v", err)
  552. return err
  553. }
  554. }
  555. if strings.Contains(line, "url") && strings.Contains(line, ".git") {
  556. originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
  557. if len(line) > len(originUrl) {
  558. originUrl += strings.Repeat(" ", len(line)-len(originUrl))
  559. }
  560. bytes := []byte(originUrl)
  561. _, err := configFile.WriteAt(bytes, pos)
  562. if err != nil {
  563. log.Error("WriteAt failed:%v", err)
  564. return err
  565. }
  566. break
  567. }
  568. pos += int64(len(line))
  569. }
  570. return nil
  571. }
  572. func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error {
  573. err := os.MkdirAll(codePath, os.ModePerm)
  574. if err != nil {
  575. log.Error("mkdir codePath failed", err.Error())
  576. return err
  577. }
  578. command := "git clone " + gitPath + " " + codePath
  579. cmd := exec.Command("/bin/bash", "-c", command)
  580. output, err := cmd.Output()
  581. log.Info(string(output))
  582. if err != nil {
  583. log.Error("exec.Command(%s) failed:%v", command, err)
  584. return err
  585. }
  586. fileName := codePath + cloudbrain.TaskInfoName
  587. f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
  588. if err != nil {
  589. log.Error("OpenFile failed", err.Error())
  590. return err
  591. }
  592. defer f.Close()
  593. data, err := json.Marshal(models.TaskInfo{
  594. Username: repo.Owner.Name,
  595. TaskName: taskName,
  596. CodeName: repo.Name,
  597. BenchmarkCategory: strings.Split(benchmarkCategory, ","),
  598. CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
  599. GpuType: gpuType,
  600. })
  601. if err != nil {
  602. log.Error("json.Marshal failed", err.Error())
  603. return err
  604. }
  605. _, err = f.Write(data)
  606. if err != nil {
  607. log.Error("WriteString failed", err.Error())
  608. return err
  609. }
  610. return nil
  611. }