Merge pull request 'cb-task' (#111) from cb-task into develop

Reviewed-by: stardust <denglf@pcl.ac.cn>

custom/conf/app.ini.sample

@@ -1046,16 +1046,15 @@ DEFAULT_QUEUE = DecompressTasksQueue
 RESULT_BACKEND = redis://localhost:6379
 
 [cloudbrain]
-HOST     = http://192.168.204.24
 USERNAME =
 PASSWORD =
-USER_CENTER_HOST = http://192.168.202.73:31441
-CLIENT_ID = 3Z377wcplxeE2qpycpjv
-CLIENT_SECRET = J5ykfVl2kcxW0H9cawSL
 REST_SERVER_HOST = http://192.168.202.73
+JOB_PATH = /datasets/minio/data/opendata/jobs/
+DEBUG_SERVER_HOST = http://192.168.202.73/
 ; cloudbrain visit opendata
 USER     = cW4cMtH24eoWPE7X
-PWD      = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC
+PWD      = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C
+JOB_TYPE = debug_openi
 
 [decompress]
 HOST     = http://192.168.207.34:39987

models/attachment.go

@@ -44,6 +44,11 @@ type Attachment struct {
 	CreatedUnix     timeutil.TimeStamp `xorm:"created"`
 }
 
+type AttachmentUsername struct {
+	Attachment `xorm:"extends"`
+	Name string
+}
+
 func (a *Attachment) AfterUpdate() {
 	if a.DatasetID > 0 {
 		datasetIsPublicCount, err := x.Where("dataset_id = ? AND is_private = ?", a.DatasetID, false).Count(new(Attachment))
@@ -348,16 +353,20 @@ func getUnDecompressAttachments(e Engine) ([]*Attachment, error) {
 	return attachments, e.Where("decompress_state = ? and dataset_id != 0 and name like '%.zip'", DecompressStateInit).Find(&attachments)
 }
 
-func GetAllPublicAttachments() ([]*Attachment, error) {
+func GetAllPublicAttachments() ([]*AttachmentUsername, error) {
 	return getAllPublicAttachments(x)
 }
 
-func getAllPublicAttachments(e Engine) ([]*Attachment, error) {
-	attachments := make([]*Attachment, 0, 10)
-	return attachments, e.Where("is_private = false and decompress_state = ?", DecompressStateDone).Find(&attachments)
+func getAllPublicAttachments(e Engine) ([]*AttachmentUsername, error) {
+	attachments := make([]*AttachmentUsername, 0, 10)
+	if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id " +
+		"= `user`.id").Where("decompress_state= ? and is_private= ?", DecompressStateDone, false).Find(&attachments); err != nil {
+		return nil, err
+	}
+	return attachments, nil
 }
 
-func GetPrivateAttachments(username string) ([]*Attachment, error) {
+func GetPrivateAttachments(username string) ([]*AttachmentUsername, error) {
 	user, err := getUserByName(x, username)
 	if err != nil {
 		log.Error("getUserByName(%s) failed:%v", username, err)
@@ -366,8 +375,28 @@ func GetPrivateAttachments(username string) ([]*Attachment, error) {
 	return getPrivateAttachments(x, user.ID)
 }
 
-func getPrivateAttachments(e Engine, userID int64) ([]*Attachment, error) {
-	attachments := make([]*Attachment, 0, 10)
-	return attachments, e.Where("uploader_id = ? and decompress_state = ?", userID, DecompressStateDone).Find(&attachments)
+func getPrivateAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) {
+	attachments := make([]*AttachmentUsername, 0, 10)
+	if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id " +
+		"= `user`.id").Where("decompress_state= ? and uploader_id= ?", DecompressStateDone, userID).Find(&attachments); err != nil {
+		return nil, err
+	}
+	return attachments, nil
+}
+
+func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
+	attachsPub, err := getAllPublicAttachments(x)
+	if err != nil {
+		log.Error("getAllPublicAttachments failed:%v", err)
+		return nil, err
+	}
+
+	attachsPri, err := getPrivateAttachments(x, userID)
+	if err != nil {
+		log.Error("getPrivateAttachments failed:%v", err)
+		return nil, err
+	}
+
+	return append(attachsPub, attachsPri...), nil
 }
 

models/cloudbrain.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"time"
+	"xorm.io/xorm"
 
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -18,17 +19,22 @@ const (
 	JobStopped   CloudbrainStatus = "STOPPED"
 	JobSucceeded CloudbrainStatus = "SUCCEEDED"
 	JobFailed    CloudbrainStatus = "FAILED"
+	JobRunning   CloudbrainStatus = "RUNNING"
 )
 
 type Cloudbrain struct {
 	ID          int64  `xorm:"pk autoincr"`
 	JobID       string `xorm:"INDEX NOT NULL"`
-	JobName     string
+	JobName     string			   `xorm:"INDEX"`
 	Status      string             `xorm:"INDEX"`
 	UserID      int64              `xorm:"INDEX"`
 	RepoID      int64              `xorm:"INDEX"`
+	SubTaskName	string			   `xorm:"INDEX"`
+	ContainerID string
+	ContainerIp string
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
+	CanDebug	bool			   `xorm:"-"`
 
 	User *User       `xorm:"-"`
 	Repo *Repository `xorm:"-"`
@@ -52,6 +58,17 @@ type TaskRole struct {
 	Command               string `json:"command"`
 	NeedIBDevice          bool   `json:"needIBDevice"`
 	IsMainRole            bool   `json:"isMainRole"`
+	UseNNI				  bool	 `json:"useNNI"`
+}
+
+type StHostPath struct {
+	Path                  string `json:"path"`
+	MountPath             string `json:"mountPath"`
+	ReadOnly              bool `json:"readOnly"`
+}
+
+type Volume struct {
+	HostPath             StHostPath `json:"hostPath"`
 }
 
 type CreateJobParams struct {
@@ -60,12 +77,13 @@ type CreateJobParams struct {
 	GpuType    string     `json:"gpuType"`
 	Image      string     `json:"image"`
 	TaskRoles  []TaskRole `json:"taskRoles"`
+	Volumes    []Volume   `json:"volumes"`
 }
 
 type CreateJobResult struct {
-	Code    string
-	Msg     string
-	Payload map[string]interface{}
+	Code    string					`json:"code"`
+	Msg     string					`json:"msg"`
+	Payload map[string]interface{}	`json:"payload"`
 }
 
 type GetJobResult struct {
@@ -74,6 +92,12 @@ type GetJobResult struct {
 	Payload map[string]interface{} `json:"payload"`
 }
 
+type GetImagesResult struct {
+	Code    string                 `json:"code"`
+	Msg     string                 `json:"msg"`
+	Payload map[string]*ImageInfo `json:"payload"`
+}
+
 type CloudbrainsOptions struct {
 	ListOptions
 	RepoID        int64 // include all repos if empty
@@ -101,6 +125,8 @@ type TaskPod struct {
 		ExitCode        int       `json:"exitCode"`
 		ExitDiagnostics string    `json:"exitDiagnostics"`
 		RetriedCount    int       `json:"retriedCount"`
+		StartTime		string
+		FinishedTime	string
 	} `json:"taskStatuses"`
 }
 
@@ -108,6 +134,8 @@ func ConvertToTaskPod(input map[string]interface{}) (TaskPod, error) {
 	data, _ := json.Marshal(input)
 	var taskPod TaskPod
 	err := json.Unmarshal(data, &taskPod)
+	taskPod.TaskStatuses[0].StartTime = time.Unix(taskPod.TaskStatuses[0].StartAt.Unix() + 8*3600, 0).UTC().Format("2006-01-02 15:04:05")
+	taskPod.TaskStatuses[0].FinishedTime = time.Unix(taskPod.TaskStatuses[0].FinishedAt.Unix() + 8*3600, 0).UTC().Format("2006-01-02 15:04:05")
 	return taskPod, err
 }
 
@@ -132,6 +160,8 @@ type JobResultPayload struct {
 		AppExitDiagnostics string      `json:"appExitDiagnostics"`
 		AppExitType        interface{} `json:"appExitType"`
 		VirtualCluster     string      `json:"virtualCluster"`
+		StartTime		   string
+		EndTime			   string
 	} `json:"jobStatus"`
 	TaskRoles map[string]interface{} `json:"taskRoles"`
 	Resource  struct {
@@ -170,9 +200,51 @@ func ConvertToJobResultPayload(input map[string]interface{}) (JobResultPayload,
 	data, _ := json.Marshal(input)
 	var jobResultPayload JobResultPayload
 	err := json.Unmarshal(data, &jobResultPayload)
+	jobResultPayload.JobStatus.StartTime = time.Unix(jobResultPayload.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05")
+	jobResultPayload.JobStatus.EndTime = time.Unix(jobResultPayload.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05")
 	return jobResultPayload, err
 }
 
+type ImagesResultPayload struct {
+	Images []struct {
+		ID       		int       `json:"id"`
+		Name          	string    `json:"name"`
+		Place           string    `json:"place"`
+		Description     string    `json:"description"`
+		Provider     	string    `json:"provider"`
+		Createtime     	string    `json:"createtime"`
+		Remark   		string    `json:"remark"`
+	} `json:"taskStatuses"`
+}
+type ImageInfo struct {
+	ID       		int       `json:"id"`
+	Name          	string    `json:"name"`
+	Place           string    `json:"place"`
+	Description     string    `json:"description"`
+	Provider     	string    `json:"provider"`
+	Createtime     	string    `json:"createtime"`
+	Remark   		string    `json:"remark"`
+	PlaceView       string
+}
+
+type CommitImageParams struct {
+	Ip    					string     `json:"ip"`
+	TaskContainerId    		string     `json:"taskContainerId"`
+	ImageTag      			string     `json:"imageTag"`
+	ImageDescription      	string     `json:"imageDescription"`
+}
+
+type CommitImageResult struct {
+	Code    string					`json:"code"`
+	Msg     string					`json:"msg"`
+	Payload map[string]interface{}	`json:"payload"`
+}
+
+type StopJobResult struct {
+	Code    string					`json:"code"`
+	Msg     string					`json:"msg"`
+}
+
 func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) {
 	sess := x.NewSession()
 	defer sess.Close()
@@ -269,3 +341,14 @@ func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err erro
 	_, err = x.Cols("status").Where("cloudbrain.job_id=?", jobID).Update(cb)
 	return
 }
+
+func UpdateJob(job *Cloudbrain) error {
+	return updateJob(x, job)
+}
+
+func updateJob(e Engine, job *Cloudbrain) error {
+	var sess *xorm.Session
+	sess = e.Where("job_id = ?", job.JobID)
+	_, err := sess.Cols("status", "container_id", "container_ip").Update(job)
+	return err
+}

modules/auth/cloudbrain.go

@@ -8,8 +8,14 @@ import (
 // CreateDatasetForm form for dataset page
 type CreateCloudBrainForm struct {
 	JobName string `form:"job_name" binding:"Required"`
-	Image   string `binding:"Required"`
-	Command string `binding:"Required"`
+	Image   string `form:"image" binding:"Required"`
+	Command string `form:"command" binding:"Required"`
+	Attachment string `form:"attachment" binding:"Required"`
+}
+
+type CommitImageCloudBrainForm struct {
+	Description string `form:"description" binding:"Required"`
+	Tag   string `form:"tag" binding:"Required"`
 }
 
 func (f *CreateCloudBrainForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {

modules/cloudbrain/cloudbrain.go

@@ -1,22 +1,39 @@
 package cloudbrain
 
 import (
+	"code.gitea.io/gitea/modules/setting"
 	"errors"
 
+	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
+)
 
-	"code.gitea.io/gitea/models"
+const (
+	Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"`
+	CodeMountPath =	"/code"
+	DataSetMountPath = "/dataset"
+	ModelMountPath = "/model"
+
+	SubTaskName = "task1"
+
+	Success = "S000"
 )
 
-func GenerateTask(ctx *context.Context, jobName, image, command string) error {
+func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath string) error {
+	dataActualPath := setting.Attachment.Minio.RealPath +
+		setting.Attachment.Minio.Bucket + "/" +
+		setting.Attachment.Minio.BasePath +
+		models.AttachmentRelativePath(uuid) +
+		uuid
 	jobResult, err := CreateJob(jobName, models.CreateJobParams{
 		JobName:    jobName,
 		RetryCount: 1,
-		GpuType:    "dgx",
+		GpuType:    setting.JobType,
 		Image:      image,
 		TaskRoles: []models.TaskRole{
 			{
-				Name:                  "task1",
+				Name:                  SubTaskName,
 				TaskNumber:            1,
 				MinSucceededTaskCount: 1,
 				MinFailedTaskCount:    1,
@@ -27,13 +44,39 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error {
 				Command:               command,
 				NeedIBDevice:          false,
 				IsMainRole:            false,
+				UseNNI:				   false,
+			},
+		},
+		Volumes: []models.Volume{
+			{
+				HostPath: models.StHostPath{
+					Path:				codePath,
+					MountPath:			CodeMountPath,
+					ReadOnly: 			false,
+				},
+			},
+			{
+				HostPath: models.StHostPath{
+					Path:				dataActualPath,
+					MountPath:			DataSetMountPath,
+					ReadOnly: 			true,
+				},
+			},
+			{
+				HostPath: models.StHostPath{
+					Path:				modelPath,
+					MountPath:			ModelMountPath,
+					ReadOnly: 			false,
+				},
 			},
 		},
 	})
 	if err != nil {
+		log.Error("CreateJob failed:", err.Error())
 		return err
 	}
-	if jobResult.Code != "S000" {
+	if jobResult.Code != Success {
+		log.Error("CreateJob(%s) failed:%s", jobName, jobResult.Msg)
 		return errors.New(jobResult.Msg)
 	}
 
@@ -44,6 +87,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error {
 		RepoID:  ctx.Repo.Repository.ID,
 		JobID:   jobID,
 		JobName: jobName,
+		SubTaskName: SubTaskName,
 	})
 
 	if err != nil {

modules/cloudbrain/resty.go

@@ -5,7 +5,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/setting"
-	resty "github.com/go-resty/resty/v2"
+	"github.com/go-resty/resty/v2"
 )
 
 var (
@@ -47,7 +47,7 @@ func loginCloudbrain() error {
 		return fmt.Errorf("resty loginCloudbrain: %s", err)
 	}
 
-	if loginResult.Code != "S000" {
+	if loginResult.Code != Success {
 		return fmt.Errorf("%s: %s", loginResult.Msg, res.String())
 	}
 
@@ -68,7 +68,7 @@ sendjob:
 		SetAuthToken(TOKEN).
 		SetBody(createJobParams).
 		SetResult(&jobResult).
-		Put(HOST + "/rest-server/api/v1/jobs/" + jobName)
+		Post(HOST + "/rest-server/api/v1/jobs/")
 
 	if err != nil {
 		return nil, fmt.Errorf("resty create job: %s", err)
@@ -80,7 +80,7 @@ sendjob:
 		goto sendjob
 	}
 
-	if jobResult.Code != "S000" {
+	if jobResult.Code != Success {
 		return &jobResult, fmt.Errorf("jobResult err: %s", res.String())
 	}
 
@@ -103,7 +103,7 @@ sendjob:
 		Get(HOST + "/rest-server/api/v1/jobs/" + jobID)
 
 	if err != nil {
-		return nil, fmt.Errorf("resty GetJob: %s", err)
+		return nil, fmt.Errorf("resty GetJob: %v", err)
 	}
 
 	if getJobResult.Code == "S401" && retry < 1 {
@@ -112,9 +112,103 @@ sendjob:
 		goto sendjob
 	}
 
-	if getJobResult.Code != "S000" {
+	if getJobResult.Code != Success {
 		return &getJobResult, fmt.Errorf("jobResult GetJob err: %s", res.String())
 	}
 
 	return &getJobResult, nil
 }
+
+func GetImages() (*models.GetImagesResult, error) {
+	checkSetting()
+	client := getRestyClient()
+	var getImagesResult models.GetImagesResult
+
+	retry := 0
+
+sendjob:
+	res, err := client.R().
+		SetHeader("Content-Type", "application/json").
+		SetAuthToken(TOKEN).
+		SetResult(&getImagesResult).
+		Get(HOST + "/rest-server/api/v1/image/list/")
+
+	if err != nil {
+		return nil, fmt.Errorf("resty GetImages: %v", err)
+	}
+
+	if getImagesResult.Code == "S401" && retry < 1 {
+		retry++
+		_ = loginCloudbrain()
+		goto sendjob
+	}
+
+	if getImagesResult.Code != Success {
+		return &getImagesResult, fmt.Errorf("getImgesResult err: %s", res.String())
+	}
+
+	return &getImagesResult, nil
+}
+
+func CommitImage(jobID string, params models.CommitImageParams) error {
+	checkSetting()
+	client := getRestyClient()
+	var result models.CommitImageResult
+
+	retry := 0
+
+sendjob:
+	res, err := client.R().
+		SetHeader("Content-Type", "application/json").
+		SetAuthToken(TOKEN).
+		SetBody(params).
+		SetResult(&result).
+		Post(HOST + "/rest-server/api/v1/jobs/" + jobID + "/commitImage")
+
+	if err != nil {
+		return fmt.Errorf("resty CommitImage: %v", err)
+	}
+
+	if result.Code == "S401" && retry < 1 {
+		retry++
+		_ = loginCloudbrain()
+		goto sendjob
+	}
+
+	if result.Code != Success {
+		return fmt.Errorf("CommitImage err: %s", res.String())
+	}
+
+	return nil
+}
+
+func StopJob(jobID string) error {
+	checkSetting()
+	client := getRestyClient()
+	var result models.StopJobResult
+
+	retry := 0
+
+sendjob:
+	res, err := client.R().
+		SetHeader("Content-Type", "application/json").
+		SetAuthToken(TOKEN).
+		SetResult(&result).
+		Delete(HOST + "/rest-server/api/v1/jobs/" + jobID)
+
+	if err != nil {
+		return fmt.Errorf("resty StopJob: %v", err)
+	}
+
+	if result.Code == "S401" && retry < 1 {
+		retry++
+		_ = loginCloudbrain()
+		goto sendjob
+	}
+
+	if result.Code != Success {
+		return fmt.Errorf("StopJob err: %s", res.String())
+	}
+
+	return nil
+}

modules/setting/cloudbrain.go

@@ -14,6 +14,6 @@ func GetCloudbrainConfig() CloudbrainLoginConfig {
 	cloudbrainSec := Cfg.Section("cloudbrain")
 	Cloudbrain.Username = cloudbrainSec.Key("USERNAME").MustString("")
 	Cloudbrain.Password = cloudbrainSec.Key("PASSWORD").MustString("")
-	Cloudbrain.Host = cloudbrainSec.Key("HOST").MustString("")
+	Cloudbrain.Host = cloudbrainSec.Key("REST_SERVER_HOST").MustString("")
 	return Cloudbrain
 }

modules/setting/setting.go

@@ -434,10 +434,10 @@ var (
 	//cloudbrain config
 	CBAuthUser				string
 	CBAuthPassword			string
-	ClientID				string
-	ClientSecret			string
-	UserCeterHost			string
 	RestServerHost			string
+	JobPath					string
+	JobType					string
+	DebugServerHost			string
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1109,10 +1109,10 @@ func NewContext() {
 	sec = Cfg.Section("cloudbrain")
 	CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X")
 	CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC")
-	ClientID = sec.Key("CLIENT_ID").MustString("3Z377wcplxeE2qpycpjv")
-	ClientSecret = sec.Key("CLIENT_SECRET").MustString("J5ykfVl2kcxW0H9cawSL")
-	UserCeterHost = sec.Key("USER_CENTER_HOST").MustString("http://192.168.202.73:31441")
 	RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73")
+	JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/")
+	DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73")
+	JobType = sec.Key("JOB_TYPE").MustString("debug_openi")
 }
 
 func loadInternalToken(sec *ini.Section) string {

options/locale/locale_zh-CN.ini

@@ -752,6 +752,7 @@ cloudbrain=云脑
 cloudbrain.new=新建任务
 cloudbrain.desc=云脑功能
 cloudbrain.cancel=取消
+cloudbrain.commit_image=提交
 
 template.items=模板选项
 template.git_content=Git数据(默认分支)

routers/api/v1/repo/cloudbrain.go

@@ -6,6 +6,7 @@
 package repo
 
 import (
+	"code.gitea.io/gitea/modules/log"
 	"net/http"
 	"time"
 
@@ -48,7 +49,7 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 
 	jobID := ctx.Params(":jobid")
 	repoID := ctx.Repo.Repository.ID
-	_, err = models.GetRepoCloudBrainByJobID(repoID, jobID)
+	job, err := models.GetRepoCloudBrainByJobID(repoID, jobID)
 	if err != nil {
 		ctx.NotFound(err)
 		return
@@ -64,8 +65,18 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 		return
 	}
 
+	taskRoles := result.TaskRoles
+	taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
+
+	job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
+	job.ContainerID = taskRes.TaskStatuses[0].ContainerID
+	job.Status = taskRes.TaskStatuses[0].State
+
 	if result.JobStatus.State != string(models.JobWaiting) {
-		go models.SetCloudbrainStatusByJobID(result.Config.JobID, models.CloudbrainStatus(result.JobStatus.State))
+		err = models.UpdateJob(job)
+		if err != nil {
+			log.Error("UpdateJob failed:", err)
+		}
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{

routers/repo/attachment.go

@@ -635,7 +635,7 @@ func QueryAllPublicDataset(ctx *context.Context){
 		return
 	}
 
-	queryDatasets(ctx, "admin", attachs)
+	queryDatasets(ctx, attachs)
 }
 
 func QueryPrivateDataset(ctx *context.Context){
@@ -650,26 +650,40 @@ func QueryPrivateDataset(ctx *context.Context){
 		return
 	}
 
-	queryDatasets(ctx, username, attachs)
+	for _, attach := range attachs {
+		attach.Name = username
+	}
+
+	queryDatasets(ctx, attachs)
 }
 
-func queryDatasets(ctx *context.Context, username string, attachs []*models.Attachment) {
+func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) {
 	var datasets []CloudBrainDataset
+	if len(attachs) == 0 {
+		log.Info("dataset is null")
+		ctx.JSON(200, map[string]string{
+			"result_code": "0",
+			"error_msg": "",
+			"data":	"",
+		})
+		return
+	}
+
 	for _, attch := range attachs {
 		has,err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID))
 		if err != nil || !has {
 			continue
 		}
 
-		datasets = append(datasets, CloudBrainDataset{attch.UUID,
-			attch.Name,
+		datasets = append(datasets, CloudBrainDataset{strconv.FormatInt(attch.ID, 10),
+			attch.Attachment.Name,
 			setting.Attachment.Minio.RealPath +
 				setting.Attachment.Minio.Bucket + "/" +
 				setting.Attachment.Minio.BasePath +
 				models.AttachmentRelativePath(attch.UUID) +
 				attch.UUID,
-			username,
-			attch.CreatedUnix.Format("2006-01-02 03:04:05")})
+			attch.Name,
+			attch.CreatedUnix.Format("2006-01-02 03:04:05 PM")})
 	}
 
 	data,err := json.Marshal(datasets)

routers/repo/cloudbrain.go

@@ -1,7 +1,11 @@
 package repo
 
 import (
+	"errors"
+	"os"
+	"os/exec"
 	"strconv"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models"
@@ -9,6 +13,7 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/cloudbrain"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -46,6 +51,15 @@ func CloudBrainIndex(ctx *context.Context) {
 		return
 	}
 
+	timestamp := time.Now().Unix()
+	for i, task := range ciTasks {
+		if task.Status == string(models.JobRunning) && (timestamp - int64(task.CreatedUnix) > 30){
+			ciTasks[i].CanDebug = true
+		} else {
+			ciTasks[i].CanDebug = false
+		}
+	}
+
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
 	pager.SetDefaultParams(ctx)
 	ctx.Data["Page"] = pager
@@ -66,10 +80,36 @@ func CloudBrainNew(ctx *context.Context) {
 	ctx.Data["PageIsCloudBrain"] = true
 
 	t := time.Now()
-	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[:5]
+	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
 	ctx.Data["job_name"] = jobName
-	ctx.Data["image"] = "192.168.202.74:5000/user-images/deepo:v2.0"
-	ctx.Data["command"] = `pip3 install jupyterlab==1.1.4;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir=\"/userhome\" --port=80  --NotebookApp.token=\"\" --LabApp.allow_origin=\"self https://cloudbrain.pcl.ac.cn\"`
+
+	result, err := cloudbrain.GetImages()
+	if err != nil {
+		ctx.Data["error"] = err.Error()
+	}
+
+	for i,payload := range result.Payload {
+		if strings.HasPrefix(result.Payload[i].Place,"192.168") {
+			result.Payload[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"): len(payload.Place)-1]
+		} else {
+			result.Payload[i].PlaceView = payload.Place
+		}
+
+	}
+
+	ctx.Data["images"] = result.Payload
+
+	attachs, err := models.GetAllUserAttachments(ctx.User.ID)
+	if err != nil {
+		ctx.ServerError("GetAllUserAttachments failed:", err)
+		return
+	}
+
+	ctx.Data["attachments"] = attachs
+	ctx.Data["command"] = cloudbrain.Command
+	ctx.Data["code_path"] = cloudbrain.CodeMountPath
+	ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
+	ctx.Data["model_path"] = cloudbrain.ModelMountPath
 	ctx.HTML(200, tplCloudBrainNew)
 }
 
@@ -78,7 +118,23 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	jobName := form.JobName
 	image := form.Image
 	command := form.Command
-	err := cloudbrain.GenerateTask(ctx, jobName, image, command)
+	uuid := form.Attachment
+	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
+	repo := ctx.Repo.Repository
+	err := downloadCode(repo, codePath)
+	if err != nil {
+		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
+		return
+	}
+
+	modelPath := setting.JobPath + jobName + "/model"
+	err = os.MkdirAll(modelPath, os.ModePerm)
+	if err != nil {
+		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
+		return
+	}
+
+	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath)
 	if err != nil {
 		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
 		return
@@ -94,7 +150,6 @@ func CloudBrainShow(ctx *context.Context) {
 	if err != nil {
 		ctx.Data["error"] = err.Error()
 	}
-	ctx.Data["task"] = task
 
 	result, err := cloudbrain.GetJob(jobID)
 	if err != nil {
@@ -105,10 +160,115 @@ func CloudBrainShow(ctx *context.Context) {
 		jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
 		ctx.Data["result"] = jobRes
 		taskRoles := jobRes.TaskRoles
-		taskRes, _ := models.ConvertToTaskPod(taskRoles["task1"].(map[string]interface{}))
+		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
 		ctx.Data["taskRes"] = taskRes
+		task.Status = taskRes.TaskStatuses[0].State
+		task.ContainerID = taskRes.TaskStatuses[0].ContainerID
+		task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
+		err = models.UpdateJob(task)
+		if err != nil {
+			ctx.Data["error"] = err.Error()
+		}
 	}
 
+	ctx.Data["task"] = task
 	ctx.Data["jobID"] = jobID
 	ctx.HTML(200, tplCloudBrainShow)
 }
+
+func CloudBrainDebug(ctx *context.Context) {
+	var jobID = ctx.Params(":jobid")
+	task, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		ctx.ServerError("GetCloudbrainByJobID failed", err)
+		return
+	}
+
+	debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
+	ctx.Redirect(debugUrl)
+}
+
+func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
+	var jobID = ctx.Params(":jobid")
+	task, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		ctx.ServerError("GetCloudbrainByJobID failed", err)
+		return
+	}
+
+	err = cloudbrain.CommitImage(jobID, models.CommitImageParams{
+		Ip:    				task.ContainerIp,
+		TaskContainerId: 	task.ContainerID,
+		ImageDescription:	form.Description,
+		ImageTag: 			form.Tag,
+	})
+	if err != nil {
+		log.Error("CommitImage(%s) failed:", task.JobName, err.Error())
+		ctx.ServerError("CommitImage failed", err)
+		return
+	}
+
+	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
+}
+
+func CloudBrainStop(ctx *context.Context) {
+	var jobID = ctx.Params(":jobid")
+	log.Info(jobID)
+	task, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		ctx.ServerError("GetCloudbrainByJobID failed", err)
+		return
+	}
+
+	if task.Status != string(models.JobRunning) {
+		log.Error("the job(%s) is not running", task.JobName)
+		ctx.ServerError("the job is not running", errors.New("the job is not running"))
+		return
+	}
+
+	err = cloudbrain.StopJob(jobID)
+	if err != nil {
+		log.Error("StopJob(%s) failed:%v", task.JobName, err.Error())
+		ctx.ServerError("StopJob failed", err)
+		return
+	}
+
+	task.Status = string(models.JobStopped)
+	err = models.UpdateJob(task)
+	if err != nil {
+		ctx.ServerError("UpdateJob failed", err)
+		return
+	}
+
+	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
+}
+
+func downloadCode(repo *models.Repository, codePath string) error {
+	/*
+	if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{
+		Bare:   true,
+		Shared: true,
+	}); err != nil {
+		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
+		return "", fmt.Errorf("Failed to clone repository: %s (%v)", repo.FullName(), err)
+	}
+
+	 */
+
+	err := os.MkdirAll(codePath, os.ModePerm)
+	if err != nil {
+		log.Error("MkdirAll failed:%v", err)
+		return err
+	}
+
+	command := "git clone " + repo.CloneLink().HTTPS + " " + codePath
+	cmd := exec.Command("/bin/bash", "-c", command)
+	output, err := cmd.Output()
+	log.Info(string(output))
+	if err != nil {
+		log.Error("exec.Command(%s) failed:%v", command, err)
+		return err
+	}
+
+	return nil
+}

routers/routes/routes.go

@@ -894,7 +894,12 @@ func RegisterRoutes(m *macaron.Macaron) {
 
 		m.Group("/cloudbrain", func() {
 			m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex)
-			m.Get("/:jobid", reqRepoCloudBrainReader, repo.CloudBrainShow)
+			m.Group("/:jobid", func() {
+				m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow)
+				m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug)
+				m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage)
+				m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop)
+			})
 			m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew)
 			m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
 		}, context.RepoRef())

templates/repo/cloudbrain/index.tmpl

@@ -35,25 +35,74 @@
 						{{range .Tasks}}
 							<div class="ui grid item">
 								<div class="row">
-									<div class="seven wide column">
+									<div class="five wide column">
 										<a class="title" href="{{$.Link}}/{{.JobID}}">
 											<span class="fitted">{{svg "octicon-tasklist" 16}}</span>
 											<span class="fitted">{{.JobName}}</span>
 										</a>
 									</div>
-									<div class="four wide column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
+									<div class="three wide column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
 										{{.Status}}
 									</div>
 									<div class="three wide column">
 										<span class="ui text center">{{svg "octicon-flame" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}</span>
 									</div>
-									<div class="two wide column">
+									<div class="one wide column">
 										<span class="ui text center clipboard">
 											<a class="title" href="{{$.Link}}/{{.JobID}}">
 												<span class="fitted">查看</span>
 											</a>
 										</span>
 									</div>
+									<div class="one wide column">
+										<span class="ui text center clipboard">
+											<a class="title" href="{{if not .CanDebug}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/debug{{end}}" style="{{if not .CanDebug}}color:#CCCCCC{{end}}">
+												<span class="fitted">调试</span>
+											</a>
+										</span>
+									</div>
+									<div class="one wide column">
+										<span class="ui text center clipboard">
+											<form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post">
+												{{$.CsrfTokenHtml}}
+												<a class="fitted" onclick="document.getElementById('stopForm-{{.JobID}}').submit();" style="{{if ne .Status "RUNNING"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">停止</a>
+											</form>
+										</span>
+									</div>
+									
+										<!-- 打开弹窗按钮 -->
+										<a class="imageBtn" style="{{if not .CanDebug}}color:#CCCCCC;cursor:pointer;pointer-events:none;{{end}}; font-size:16px; font-weight:bold" value="{{.CanDebug}}">提交镜像</a>
+										<!-- 弹窗 -->
+										<div id="imageModal" class="modal">
+										<!-- 弹窗内容 -->
+										<div class="modal-content">
+											<span class="close">&times;</span>
+											<form id="commitImageForm" action="{{$.Link}}/{{.JobID}}/commit_image" method="post">
+												{{$.CsrfTokenHtml}}
+												<p>提交任务镜像</p>
+												<div class="ui divider"></div>
+												<div class="inline required field dis">
+													<label>镜像标签：</label>
+													<input name="tag" id="image_tag" tabindex="3" autofocus required maxlength="255" style="width:75%">
+												</div>
+												<div class="inline required field" style="position:relative;height:180px;">
+													<div style="height:20px;width:75px;">
+														<label>镜像描述：</label>
+													</div>
+													<div style="position:absolute;left:75px;top:0;width:75%">
+														<textarea name="description" rows="10" style="width:100%"></textarea>
+													</div>
+												</div>
+												<div class="ui divider"></div>
+												<div class="inline field">
+													<label></label>
+													<button class="ui green button">
+														{{$.i18n.Tr "repo.cloudbrain.commit_image"}}
+													</button>
+												</div>
+											</form>
+										</div>
+									</div>
 								</div>
 							</div>
 						{{end}}
@@ -70,7 +119,7 @@ $( document ).ready(function() {
 	$( ".job-status" ).each(( index, job ) => {
 		const jobID = job.dataset.jobid;
 		const repoPath = job.dataset.repopath;
-		if (job.textContent.trim()  != 'WAITING') {
+		if (job.textContent.trim()  == 'STOPPED') {
 			return
 		}
 
@@ -84,4 +133,83 @@ $( document ).ready(function() {
 		});
 	});
 });
+
+// 获取弹窗
+var modal = document.getElementById('imageModal');
+ 
+// 打开弹窗的按钮对象
+var btns = document.getElementsByClassName("imageBtn");
+ 
+// 获取 <span> 元素，用于关闭弹窗
+var spans = document.getElementsByClassName('close');
+ 
+// 点击按钮打开弹窗
+for(i=0;i<btns.length;i++){
+	btns[i].onclick = function() {
+		modal.style.display = "block";
+	}
+}
+ 
+// 点击 <span> (x), 关闭弹窗
+for(i=0;i<spans.length;i++){
+	spans[i].onclick = function() {
+		
+		modal.style.display = "none";
+	}
+} 
+
+// 在用户点击其他地方时，关闭弹窗
+window.onclick = function(event) {
+    if (event.target == modal) {
+        modal.style.display = "none";
+    }
+}
+
 </script>
+
+
+<style>
+/* 弹窗 (background) */
+.modal {
+    display: none; /* 默认隐藏 */
+    position: fixed; /* 固定定位 */
+    z-index: 1; /* 设置在顶层 */
+    left: 0;
+    top: 0;
+    width: 100%; 
+    height: 100%;
+    overflow: auto; 
+    background-color: rgb(0,0,0); 
+    background-color: rgba(0,0,0,0.4); 
+}
+ 
+/* 弹窗内容 */
+.modal-content {
+    background-color: #fefefe;
+    margin: 15% auto; 
+    padding: 20px;
+    border: 1px solid #888;
+    width: 25%; 
+}
+ 
+/* 关闭按钮 */
+.close {
+    color: #aaa;
+    float: right;
+    font-size: 28px;
+    font-weight: bold;
+}
+ 
+.close:hover,
+.close:focus {
+    color: black;
+    text-decoration: none;
+    cursor: pointer;
+}
+
+.dis{
+	margin-bottom:20px;
+}
+
+
+</style>

templates/repo/cloudbrain/new.tmpl

@@ -18,18 +18,42 @@
           <br>
           <div class="inline required field">
             <label>镜像</label>
-            <input name="image" id="cloudbrain_image" placeholder="输入镜像" value="{{.image}}" tabindex="3" autofocus required maxlength="255">
+            <select id="cloudbrain_image" placeholder="选择镜像" style='width:385px' name="image">
+              {{range .images}}
+                <option name="image" value="{{.Place}}">{{.PlaceView}}</option>
+              {{end}}
+            </select>
+          </div>
+          <div class="inline required field">
+            <label>数据集(只有zip格式的数据集才能发起云脑任务)</label>
+            <select id="cloudbrain_dataset" placeholder="选择数据集" style='width:385px' name="attachment">
+              {{range .attachments}}
+                <option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
+              {{end}}
+            </select>
+          </div>
+          <div class="inline required field">
+            <label>数据集存放路径</label>
+            <input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
+          </div>
+          <div class="inline required field">
+            <label>模型存放路径</label>
+            <input name="model_path" id="cloudbrain_model_path" value="{{.model_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
+          </div>
+          <div class="inline required field">
+            <label>代码存放路径</label>
+            <input name="code_path" id="cloudbrain_code_path" value="{{.code_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
           </div>
           <div class="inline required field">
             <label>启动命令</label>
-            <textarea name="command" rows="10">{{.command}}</textarea>
+            <textarea name="command" rows="10" readonly="readonly">{{.command}}</textarea>
           </div>
           <div class="inline field">
             <label></label>
             <button class="ui green button">
               {{.i18n.Tr "repo.cloudbrain.new"}}
             </button>
-            <a class="ui button" href="/">Cancel</a>
+            <a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
           </div>
         </div>
       </form>

templates/repo/cloudbrain/show.tmpl

@@ -26,11 +26,11 @@
                 </tr>
                 <tr>
                   <td> 开始时间 </td>
-                  <td>{{.StartAt}}</td>
+                  <td>{{.StartTime}}</td>
                 </tr>
                 <tr>
                   <td> 结束时间 </td>
-                  <td>{{.FinishedAt}}</td>
+                  <td>{{.FinishedTime}}</td>
                 </tr>
                 <tr>
                   <td> ExitCode </td>
@@ -78,11 +78,11 @@
                 </tr>
                 <tr>
                   <td> 开始时间 </td>
-                  <td>{{.JobStatus.CreatedTime}}</td>
+                  <td>{{.JobStatus.StartTime}}</td>
                 </tr>
                 <tr>
                   <td> 结束时间 </td>
-                  <td>{{.JobStatus.CompletedTime}}</td>
+                  <td>{{.JobStatus.EndTime}}</td>
                 </tr>
                 <tr>
                   <td> ExitCode </td>