diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index bc30ac5c2..c469fee7b 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1046,16 +1046,15 @@ DEFAULT_QUEUE = DecompressTasksQueue RESULT_BACKEND = redis://localhost:6379 [cloudbrain] -HOST = http://192.168.204.24 USERNAME = PASSWORD = -USER_CENTER_HOST = http://192.168.202.73:31441 -CLIENT_ID = 3Z377wcplxeE2qpycpjv -CLIENT_SECRET = J5ykfVl2kcxW0H9cawSL REST_SERVER_HOST = http://192.168.202.73 +JOB_PATH = /datasets/minio/data/opendata/jobs/ +DEBUG_SERVER_HOST = http://192.168.202.73/ ; cloudbrain visit opendata USER = cW4cMtH24eoWPE7X -PWD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC +PWD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C +JOB_TYPE = debug_openi [decompress] HOST = http://192.168.207.34:39987 diff --git a/docs/开源社区与云脑平台对接方案.docx b/docs/开源社区与云脑平台对接方案.docx index 8db5c651a..20966e992 100755 Binary files a/docs/开源社区与云脑平台对接方案.docx and b/docs/开源社区与云脑平台对接方案.docx differ diff --git a/models/attachment.go b/models/attachment.go index 3d4e4d136..3ba4112dc 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -44,6 +44,11 @@ type Attachment struct { CreatedUnix timeutil.TimeStamp `xorm:"created"` } +type AttachmentUsername struct { + Attachment `xorm:"extends"` + Name string +} + func (a *Attachment) AfterUpdate() { if a.DatasetID > 0 { datasetIsPublicCount, err := x.Where("dataset_id = ? AND is_private = ?", a.DatasetID, false).Count(new(Attachment)) @@ -348,16 +353,20 @@ func getUnDecompressAttachments(e Engine) ([]*Attachment, error) { return attachments, e.Where("decompress_state = ? and dataset_id != 0 and name like '%.zip'", DecompressStateInit).Find(&attachments) } -func GetAllPublicAttachments() ([]*Attachment, error) { +func GetAllPublicAttachments() ([]*AttachmentUsername, error) { return getAllPublicAttachments(x) } -func getAllPublicAttachments(e Engine) ([]*Attachment, error) { - attachments := make([]*Attachment, 0, 10) - return attachments, e.Where("is_private = false and decompress_state = ?", DecompressStateDone).Find(&attachments) +func getAllPublicAttachments(e Engine) ([]*AttachmentUsername, error) { + attachments := make([]*AttachmentUsername, 0, 10) + if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id " + + "= `user`.id").Where("decompress_state= ? and is_private= ?", DecompressStateDone, false).Find(&attachments); err != nil { + return nil, err + } + return attachments, nil } -func GetPrivateAttachments(username string) ([]*Attachment, error) { +func GetPrivateAttachments(username string) ([]*AttachmentUsername, error) { user, err := getUserByName(x, username) if err != nil { log.Error("getUserByName(%s) failed:%v", username, err) @@ -366,8 +375,28 @@ func GetPrivateAttachments(username string) ([]*Attachment, error) { return getPrivateAttachments(x, user.ID) } -func getPrivateAttachments(e Engine, userID int64) ([]*Attachment, error) { - attachments := make([]*Attachment, 0, 10) - return attachments, e.Where("uploader_id = ? and decompress_state = ?", userID, DecompressStateDone).Find(&attachments) +func getPrivateAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) { + attachments := make([]*AttachmentUsername, 0, 10) + if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id " + + "= `user`.id").Where("decompress_state= ? and uploader_id= ?", DecompressStateDone, userID).Find(&attachments); err != nil { + return nil, err + } + return attachments, nil +} + +func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) { + attachsPub, err := getAllPublicAttachments(x) + if err != nil { + log.Error("getAllPublicAttachments failed:%v", err) + return nil, err + } + + attachsPri, err := getPrivateAttachments(x, userID) + if err != nil { + log.Error("getPrivateAttachments failed:%v", err) + return nil, err + } + + return append(attachsPub, attachsPri...), nil } diff --git a/models/cloudbrain.go b/models/cloudbrain.go old mode 100644 new mode 100755 index 997cf41cf..8cf2132f9 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "time" + "xorm.io/xorm" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -18,17 +19,22 @@ const ( JobStopped CloudbrainStatus = "STOPPED" JobSucceeded CloudbrainStatus = "SUCCEEDED" JobFailed CloudbrainStatus = "FAILED" + JobRunning CloudbrainStatus = "RUNNING" ) type Cloudbrain struct { ID int64 `xorm:"pk autoincr"` JobID string `xorm:"INDEX NOT NULL"` - JobName string + JobName string `xorm:"INDEX"` Status string `xorm:"INDEX"` UserID int64 `xorm:"INDEX"` RepoID int64 `xorm:"INDEX"` + SubTaskName string `xorm:"INDEX"` + ContainerID string + ContainerIp string CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + CanDebug bool `xorm:"-"` User *User `xorm:"-"` Repo *Repository `xorm:"-"` @@ -52,6 +58,17 @@ type TaskRole struct { Command string `json:"command"` NeedIBDevice bool `json:"needIBDevice"` IsMainRole bool `json:"isMainRole"` + UseNNI bool `json:"useNNI"` +} + +type StHostPath struct { + Path string `json:"path"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly"` +} + +type Volume struct { + HostPath StHostPath `json:"hostPath"` } type CreateJobParams struct { @@ -60,12 +77,13 @@ type CreateJobParams struct { GpuType string `json:"gpuType"` Image string `json:"image"` TaskRoles []TaskRole `json:"taskRoles"` + Volumes []Volume `json:"volumes"` } type CreateJobResult struct { - Code string - Msg string - Payload map[string]interface{} + Code string `json:"code"` + Msg string `json:"msg"` + Payload map[string]interface{} `json:"payload"` } type GetJobResult struct { @@ -74,6 +92,12 @@ type GetJobResult struct { Payload map[string]interface{} `json:"payload"` } +type GetImagesResult struct { + Code string `json:"code"` + Msg string `json:"msg"` + Payload map[string]*ImageInfo `json:"payload"` +} + type CloudbrainsOptions struct { ListOptions RepoID int64 // include all repos if empty @@ -101,6 +125,8 @@ type TaskPod struct { ExitCode int `json:"exitCode"` ExitDiagnostics string `json:"exitDiagnostics"` RetriedCount int `json:"retriedCount"` + StartTime string + FinishedTime string } `json:"taskStatuses"` } @@ -108,6 +134,8 @@ func ConvertToTaskPod(input map[string]interface{}) (TaskPod, error) { data, _ := json.Marshal(input) var taskPod TaskPod err := json.Unmarshal(data, &taskPod) + taskPod.TaskStatuses[0].StartTime = time.Unix(taskPod.TaskStatuses[0].StartAt.Unix() + 8*3600, 0).UTC().Format("2006-01-02 15:04:05") + taskPod.TaskStatuses[0].FinishedTime = time.Unix(taskPod.TaskStatuses[0].FinishedAt.Unix() + 8*3600, 0).UTC().Format("2006-01-02 15:04:05") return taskPod, err } @@ -132,6 +160,8 @@ type JobResultPayload struct { AppExitDiagnostics string `json:"appExitDiagnostics"` AppExitType interface{} `json:"appExitType"` VirtualCluster string `json:"virtualCluster"` + StartTime string + EndTime string } `json:"jobStatus"` TaskRoles map[string]interface{} `json:"taskRoles"` Resource struct { @@ -170,9 +200,51 @@ func ConvertToJobResultPayload(input map[string]interface{}) (JobResultPayload, data, _ := json.Marshal(input) var jobResultPayload JobResultPayload err := json.Unmarshal(data, &jobResultPayload) + jobResultPayload.JobStatus.StartTime = time.Unix(jobResultPayload.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05") + jobResultPayload.JobStatus.EndTime = time.Unix(jobResultPayload.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05") return jobResultPayload, err } +type ImagesResultPayload struct { + Images []struct { + ID int `json:"id"` + Name string `json:"name"` + Place string `json:"place"` + Description string `json:"description"` + Provider string `json:"provider"` + Createtime string `json:"createtime"` + Remark string `json:"remark"` + } `json:"taskStatuses"` +} +type ImageInfo struct { + ID int `json:"id"` + Name string `json:"name"` + Place string `json:"place"` + Description string `json:"description"` + Provider string `json:"provider"` + Createtime string `json:"createtime"` + Remark string `json:"remark"` + PlaceView string +} + +type CommitImageParams struct { + Ip string `json:"ip"` + TaskContainerId string `json:"taskContainerId"` + ImageTag string `json:"imageTag"` + ImageDescription string `json:"imageDescription"` +} + +type CommitImageResult struct { + Code string `json:"code"` + Msg string `json:"msg"` + Payload map[string]interface{} `json:"payload"` +} + +type StopJobResult struct { + Code string `json:"code"` + Msg string `json:"msg"` +} + func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { sess := x.NewSession() defer sess.Close() @@ -269,3 +341,14 @@ func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err erro _, err = x.Cols("status").Where("cloudbrain.job_id=?", jobID).Update(cb) return } + +func UpdateJob(job *Cloudbrain) error { + return updateJob(x, job) +} + +func updateJob(e Engine, job *Cloudbrain) error { + var sess *xorm.Session + sess = e.Where("job_id = ?", job.JobID) + _, err := sess.Cols("status", "container_id", "container_ip").Update(job) + return err +} diff --git a/models/dataset.go b/models/dataset.go old mode 100644 new mode 100755 diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go old mode 100644 new mode 100755 index 938bf087e..b7b59ce63 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -8,8 +8,14 @@ import ( // CreateDatasetForm form for dataset page type CreateCloudBrainForm struct { JobName string `form:"job_name" binding:"Required"` - Image string `binding:"Required"` - Command string `binding:"Required"` + Image string `form:"image" binding:"Required"` + Command string `form:"command" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` +} + +type CommitImageCloudBrainForm struct { + Description string `form:"description" binding:"Required"` + Tag string `form:"tag" binding:"Required"` } func (f *CreateCloudBrainForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 96a6f2957..52057f0b8 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,22 +1,39 @@ package cloudbrain import ( + "code.gitea.io/gitea/modules/setting" "errors" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) - "code.gitea.io/gitea/models" +const ( + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + CodeMountPath = "/code" + DataSetMountPath = "/dataset" + ModelMountPath = "/model" + + SubTaskName = "task1" + + Success = "S000" ) -func GenerateTask(ctx *context.Context, jobName, image, command string) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath string) error { + dataActualPath := setting.Attachment.Minio.RealPath + + setting.Attachment.Minio.Bucket + "/" + + setting.Attachment.Minio.BasePath + + models.AttachmentRelativePath(uuid) + + uuid jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, - GpuType: "dgx", + GpuType: setting.JobType, Image: image, TaskRoles: []models.TaskRole{ { - Name: "task1", + Name: SubTaskName, TaskNumber: 1, MinSucceededTaskCount: 1, MinFailedTaskCount: 1, @@ -27,13 +44,39 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error { Command: command, NeedIBDevice: false, IsMainRole: false, + UseNNI: false, + }, + }, + Volumes: []models.Volume{ + { + HostPath: models.StHostPath{ + Path: codePath, + MountPath: CodeMountPath, + ReadOnly: false, + }, + }, + { + HostPath: models.StHostPath{ + Path: dataActualPath, + MountPath: DataSetMountPath, + ReadOnly: true, + }, + }, + { + HostPath: models.StHostPath{ + Path: modelPath, + MountPath: ModelMountPath, + ReadOnly: false, + }, }, }, }) if err != nil { + log.Error("CreateJob failed:", err.Error()) return err } - if jobResult.Code != "S000" { + if jobResult.Code != Success { + log.Error("CreateJob(%s) failed:%s", jobName, jobResult.Msg) return errors.New(jobResult.Msg) } @@ -44,6 +87,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error { RepoID: ctx.Repo.Repository.ID, JobID: jobID, JobName: jobName, + SubTaskName: SubTaskName, }) if err != nil { diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go old mode 100644 new mode 100755 index 88134bab7..a58e31e20 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -5,7 +5,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" - resty "github.com/go-resty/resty/v2" + "github.com/go-resty/resty/v2" ) var ( @@ -47,7 +47,7 @@ func loginCloudbrain() error { return fmt.Errorf("resty loginCloudbrain: %s", err) } - if loginResult.Code != "S000" { + if loginResult.Code != Success { return fmt.Errorf("%s: %s", loginResult.Msg, res.String()) } @@ -68,7 +68,7 @@ sendjob: SetAuthToken(TOKEN). SetBody(createJobParams). SetResult(&jobResult). - Put(HOST + "/rest-server/api/v1/jobs/" + jobName) + Post(HOST + "/rest-server/api/v1/jobs/") if err != nil { return nil, fmt.Errorf("resty create job: %s", err) @@ -80,7 +80,7 @@ sendjob: goto sendjob } - if jobResult.Code != "S000" { + if jobResult.Code != Success { return &jobResult, fmt.Errorf("jobResult err: %s", res.String()) } @@ -103,7 +103,7 @@ sendjob: Get(HOST + "/rest-server/api/v1/jobs/" + jobID) if err != nil { - return nil, fmt.Errorf("resty GetJob: %s", err) + return nil, fmt.Errorf("resty GetJob: %v", err) } if getJobResult.Code == "S401" && retry < 1 { @@ -112,9 +112,103 @@ sendjob: goto sendjob } - if getJobResult.Code != "S000" { + if getJobResult.Code != Success { return &getJobResult, fmt.Errorf("jobResult GetJob err: %s", res.String()) } return &getJobResult, nil } + +func GetImages() (*models.GetImagesResult, error) { + checkSetting() + client := getRestyClient() + var getImagesResult models.GetImagesResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&getImagesResult). + Get(HOST + "/rest-server/api/v1/image/list/") + + if err != nil { + return nil, fmt.Errorf("resty GetImages: %v", err) + } + + if getImagesResult.Code == "S401" && retry < 1 { + retry++ + _ = loginCloudbrain() + goto sendjob + } + + if getImagesResult.Code != Success { + return &getImagesResult, fmt.Errorf("getImgesResult err: %s", res.String()) + } + + return &getImagesResult, nil +} + +func CommitImage(jobID string, params models.CommitImageParams) error { + checkSetting() + client := getRestyClient() + var result models.CommitImageResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(params). + SetResult(&result). + Post(HOST + "/rest-server/api/v1/jobs/" + jobID + "/commitImage") + + if err != nil { + return fmt.Errorf("resty CommitImage: %v", err) + } + + if result.Code == "S401" && retry < 1 { + retry++ + _ = loginCloudbrain() + goto sendjob + } + + if result.Code != Success { + return fmt.Errorf("CommitImage err: %s", res.String()) + } + + return nil +} + +func StopJob(jobID string) error { + checkSetting() + client := getRestyClient() + var result models.StopJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/rest-server/api/v1/jobs/" + jobID) + + if err != nil { + return fmt.Errorf("resty StopJob: %v", err) + } + + if result.Code == "S401" && retry < 1 { + retry++ + _ = loginCloudbrain() + goto sendjob + } + + if result.Code != Success { + return fmt.Errorf("StopJob err: %s", res.String()) + } + + return nil +} diff --git a/modules/setting/cloudbrain.go b/modules/setting/cloudbrain.go old mode 100644 new mode 100755 index 71d59c697..c0ab3b275 --- a/modules/setting/cloudbrain.go +++ b/modules/setting/cloudbrain.go @@ -14,6 +14,6 @@ func GetCloudbrainConfig() CloudbrainLoginConfig { cloudbrainSec := Cfg.Section("cloudbrain") Cloudbrain.Username = cloudbrainSec.Key("USERNAME").MustString("") Cloudbrain.Password = cloudbrainSec.Key("PASSWORD").MustString("") - Cloudbrain.Host = cloudbrainSec.Key("HOST").MustString("") + Cloudbrain.Host = cloudbrainSec.Key("REST_SERVER_HOST").MustString("") return Cloudbrain } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index af902cb79..3fa6403a1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -434,10 +434,10 @@ var ( //cloudbrain config CBAuthUser string CBAuthPassword string - ClientID string - ClientSecret string - UserCeterHost string RestServerHost string + JobPath string + JobType string + DebugServerHost string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1109,10 +1109,10 @@ func NewContext() { sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") - ClientID = sec.Key("CLIENT_ID").MustString("3Z377wcplxeE2qpycpjv") - ClientSecret = sec.Key("CLIENT_SECRET").MustString("J5ykfVl2kcxW0H9cawSL") - UserCeterHost = sec.Key("USER_CENTER_HOST").MustString("http://192.168.202.73:31441") RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73") + JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/") + DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") + JobType = sec.Key("JOB_TYPE").MustString("debug_openi") } func loadInternalToken(sec *ini.Section) string { diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2e56fbb06..0398d5655 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -752,6 +752,7 @@ cloudbrain=云脑 cloudbrain.new=新建任务 cloudbrain.desc=云脑功能 cloudbrain.cancel=取消 +cloudbrain.commit_image=提交 template.items=模板选项 template.git_content=Git数据(默认分支) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go old mode 100644 new mode 100755 index 56d41830b..f4364f1e4 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -6,6 +6,7 @@ package repo import ( + "code.gitea.io/gitea/modules/log" "net/http" "time" @@ -48,7 +49,7 @@ func GetCloudbrainTask(ctx *context.APIContext) { jobID := ctx.Params(":jobid") repoID := ctx.Repo.Repository.ID - _, err = models.GetRepoCloudBrainByJobID(repoID, jobID) + job, err := models.GetRepoCloudBrainByJobID(repoID, jobID) if err != nil { ctx.NotFound(err) return @@ -64,8 +65,18 @@ func GetCloudbrainTask(ctx *context.APIContext) { return } + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State + if result.JobStatus.State != string(models.JobWaiting) { - go models.SetCloudbrainStatusByJobID(result.Config.JobID, models.CloudbrainStatus(result.JobStatus.State)) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } } ctx.JSON(http.StatusOK, map[string]interface{}{ diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3881ff733..f474f88d6 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -635,7 +635,7 @@ func QueryAllPublicDataset(ctx *context.Context){ return } - queryDatasets(ctx, "admin", attachs) + queryDatasets(ctx, attachs) } func QueryPrivateDataset(ctx *context.Context){ @@ -650,26 +650,40 @@ func QueryPrivateDataset(ctx *context.Context){ return } - queryDatasets(ctx, username, attachs) + for _, attach := range attachs { + attach.Name = username + } + + queryDatasets(ctx, attachs) } -func queryDatasets(ctx *context.Context, username string, attachs []*models.Attachment) { +func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) { var datasets []CloudBrainDataset + if len(attachs) == 0 { + log.Info("dataset is null") + ctx.JSON(200, map[string]string{ + "result_code": "0", + "error_msg": "", + "data": "", + }) + return + } + for _, attch := range attachs { has,err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID)) if err != nil || !has { continue } - datasets = append(datasets, CloudBrainDataset{attch.UUID, - attch.Name, + datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10), + attch.Attachment.Name, setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + models.AttachmentRelativePath(attch.UUID) + attch.UUID, - username, - attch.CreatedUnix.Format("2006-01-02 03:04:05")}) + attch.Name, + attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")}) } data,err := json.Marshal(datasets) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go old mode 100644 new mode 100755 index e258eadd7..6742864c6 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1,7 +1,11 @@ package repo import ( + "errors" + "os" + "os/exec" "strconv" + "strings" "time" "code.gitea.io/gitea/models" @@ -9,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -46,6 +51,15 @@ func CloudBrainIndex(ctx *context.Context) { return } + timestamp := time.Now().Unix() + for i, task := range ciTasks { + if task.Status == string(models.JobRunning) && (timestamp - int64(task.CreatedUnix) > 30){ + ciTasks[i].CanDebug = true + } else { + ciTasks[i].CanDebug = false + } + } + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager @@ -66,10 +80,36 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true t := time.Now() - var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[:5] + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = jobName - ctx.Data["image"] = "192.168.202.74:5000/user-images/deepo:v2.0" - ctx.Data["command"] = `pip3 install jupyterlab==1.1.4;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir=\"/userhome\" --port=80 --NotebookApp.token=\"\" --LabApp.allow_origin=\"self https://cloudbrain.pcl.ac.cn\"` + + result, err := cloudbrain.GetImages() + if err != nil { + ctx.Data["error"] = err.Error() + } + + for i,payload := range result.Payload { + if strings.HasPrefix(result.Payload[i].Place,"192.168") { + result.Payload[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"): len(payload.Place)-1] + } else { + result.Payload[i].PlaceView = payload.Place + } + + } + + ctx.Data["images"] = result.Payload + + attachs, err := models.GetAllUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return + } + + ctx.Data["attachments"] = attachs + ctx.Data["command"] = cloudbrain.Command + ctx.Data["code_path"] = cloudbrain.CodeMountPath + ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath + ctx.Data["model_path"] = cloudbrain.ModelMountPath ctx.HTML(200, tplCloudBrainNew) } @@ -78,7 +118,23 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { jobName := form.JobName image := form.Image command := form.Command - err := cloudbrain.GenerateTask(ctx, jobName, image, command) + uuid := form.Attachment + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + repo := ctx.Repo.Repository + err := downloadCode(repo, codePath) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + + modelPath := setting.JobPath + jobName + "/model" + err = os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath) if err != nil { ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return @@ -94,7 +150,6 @@ func CloudBrainShow(ctx *context.Context) { if err != nil { ctx.Data["error"] = err.Error() } - ctx.Data["task"] = task result, err := cloudbrain.GetJob(jobID) if err != nil { @@ -105,10 +160,115 @@ func CloudBrainShow(ctx *context.Context) { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) ctx.Data["result"] = jobRes taskRoles := jobRes.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles["task1"].(map[string]interface{})) + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) ctx.Data["taskRes"] = taskRes + task.Status = taskRes.TaskStatuses[0].State + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + } } + ctx.Data["task"] = task ctx.Data["jobID"] = jobID ctx.HTML(200, tplCloudBrainShow) } + +func CloudBrainDebug(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + ctx.Redirect(debugUrl) +} + +func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + err = cloudbrain.CommitImage(jobID, models.CommitImageParams{ + Ip: task.ContainerIp, + TaskContainerId: task.ContainerID, + ImageDescription: form.Description, + ImageTag: form.Tag, + }) + if err != nil { + log.Error("CommitImage(%s) failed:", task.JobName, err.Error()) + ctx.ServerError("CommitImage failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") +} + +func CloudBrainStop(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + log.Info(jobID) + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobRunning) { + log.Error("the job(%s) is not running", task.JobName) + ctx.ServerError("the job is not running", errors.New("the job is not running")) + return + } + + err = cloudbrain.StopJob(jobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("StopJob failed", err) + return + } + + task.Status = string(models.JobStopped) + err = models.UpdateJob(task) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") +} + +func downloadCode(repo *models.Repository, codePath string) error { + /* + if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{ + Bare: true, + Shared: true, + }); err != nil { + log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) + return "", fmt.Errorf("Failed to clone repository: %s (%v)", repo.FullName(), err) + } + + */ + + err := os.MkdirAll(codePath, os.ModePerm) + if err != nil { + log.Error("MkdirAll failed:%v", err) + return err + } + + command := "git clone " + repo.CloneLink().HTTPS + " " + codePath + cmd := exec.Command("/bin/bash", "-c", command) + output, err := cmd.Output() + log.Info(string(output)) + if err != nil { + log.Error("exec.Command(%s) failed:%v", command, err) + return err + } + + return nil +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 8d722cfe2..814af1fd9 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -894,7 +894,12 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrain", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) - m.Get("/:jobid", reqRepoCloudBrainReader, repo.CloudBrainShow) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) + m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) + m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) + m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) + }) m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl old mode 100644 new mode 100755 index 05e98506e..6a4ba1f50 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -35,25 +35,74 @@ {{range .Tasks}}
-
+
{{svg "octicon-tasklist" 16}} {{.JobName}}
-
+
{{.Status}}
{{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}
-
+
查看
+
+ + + 调试 + + +
+
+ +
+ {{$.CsrfTokenHtml}} + 停止 +
+ +
+ + + 提交镜像 + +
{{end}} @@ -70,7 +119,7 @@ $( document ).ready(function() { $( ".job-status" ).each(( index, job ) => { const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() != 'WAITING') { + if (job.textContent.trim() == 'STOPPED') { return } @@ -84,4 +133,83 @@ $( document ).ready(function() { }); }); }); + +// 获取弹窗 +var modal = document.getElementById('imageModal'); + +// 打开弹窗的按钮对象 +var btns = document.getElementsByClassName("imageBtn"); + +// 获取 元素，用于关闭弹窗 +var spans = document.getElementsByClassName('close'); + +// 点击按钮打开弹窗 +for(i=0;i (x), 关闭弹窗 +for(i=0;i + + + \ No newline at end of file diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl old mode 100644 new mode 100755 index 944d7b25f..f2b5ed6f8 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -18,18 +18,42 @@
- + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
- +
- Cancel + {{.i18n.Tr "repo.cloudbrain.cancel"}}
diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl old mode 100644 new mode 100755 index ba62ccca9..fe4ec7ab3 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -26,11 +26,11 @@ 开始时间 - {{.StartAt}} + {{.StartTime}} 结束时间 - {{.FinishedAt}} + {{.FinishedTime}} ExitCode @@ -78,11 +78,11 @@ 开始时间 - {{.JobStatus.CreatedTime}} + {{.JobStatus.StartTime}} 结束时间 - {{.JobStatus.CompletedTime}} + {{.JobStatus.EndTime}} ExitCode