Merge pull request 'NPU调试任务优化' (#1536) from opt-notebook into V20220228

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/1536
Reviewed-by: ychao_1983 <ychao_1983@sina.com>

models/cloudbrain.go

@@ -102,7 +102,7 @@ type Cloudbrain struct {
 	UpdatedUnix          timeutil.TimeStamp `xorm:"INDEX updated"`
 	Duration             int64
 	TrainJobDuration     string
-	Image                string    //GPU镜像名称
+	Image                string    //镜像名称
 	GpuQueue             string    //GPU类型即GPU队列
 	ResourceSpecId       int       //GPU规格id
 	DeletedAt            time.Time `xorm:"deleted"`
@@ -449,6 +449,16 @@ type FlavorInfo struct {
 	Desc  string `json:"desc"`
 }
 
+type ImageInfosModelArts struct {
+	ImageInfo []*ImageInfoModelArts `json:"image_info"`
+}
+
+type ImageInfoModelArts struct {
+	Id    string `json:"id"`
+	Value string `json:"value"`
+	Desc  string `json:"desc"`
+}
+
 type PoolInfos struct {
 	PoolInfo []*PoolInfo `json:"pool_info"`
 }

modules/auth/modelarts.go

@@ -19,7 +19,8 @@ type CreateModelArtsNotebookForm struct {
 	JobName     string `form:"job_name" binding:"Required"`
 	Attachment  string `form:"attachment"`
 	Description string `form:"description"`
-	Flavor      string `form:"flavor"`
+	Flavor      string `form:"flavor" binding:"Required"`
+	ImageId     string `form:"image_id" binding:"Required"`
 }
 
 func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {

modules/modelarts/modelarts.go

@@ -2,6 +2,7 @@ package modelarts
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"path"
 	"strconv"
@@ -16,8 +17,8 @@ import (
 
 const (
 	//notebook
-	storageTypeOBS   = "obs"
-	autoStopDuration = 4 * 60 * 60
+	storageTypeOBS     = "obs"
+	autoStopDuration   = 4 * 60 * 60
 	autoStopDurationMs = 4 * 60 * 60 * 1000
 
 	DataSetMountPath = "/home/ma-user/work"
@@ -64,6 +65,7 @@ const (
 var (
 	poolInfos   *models.PoolInfos
 	FlavorInfos *models.FlavorInfos
+	ImageInfos  *models.ImageInfosModelArts
 )
 
 type GenerateTrainJobReq struct {
@@ -264,31 +266,38 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin
 	return nil
 }
 
-func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor string) error {
+func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor, imageId string) error {
 	if poolInfos == nil {
 		json.Unmarshal([]byte(setting.PoolInfos), &poolInfos)
 	}
+
+	imageName, err := GetNotebookImageName(imageId)
+	if err != nil {
+		log.Error("GetNotebookImageName failed: %v", err.Error())
+		return err
+	}
+
 	jobResult, err := createNotebook2(models.CreateNotebook2Params{
 		JobName:     jobName,
 		Description: description,
 		Flavor:      flavor,
 		Duration:    autoStopDurationMs,
-		ImageID:     "59a6e9f5-93c0-44dd-85b0-82f390c5d53a",
+		ImageID:     imageId,
 		PoolID:      poolInfos.PoolInfo[0].PoolId,
 		Feature:     models.NotebookFeature,
-		Volume:      models.VolumeReq{
-			Capacity: 	100,
-			Category: 	models.EVSCategory,
-			Ownership:  models.ManagedOwnership,
+		Volume: models.VolumeReq{
+			Capacity:  setting.Capacity,
+			Category:  models.EVSCategory,
+			Ownership: models.ManagedOwnership,
 		},
-		WorkspaceID:  "0",
+		WorkspaceID: "0",
 	})
 	if err != nil {
 		log.Error("createNotebook2 failed: %v", err.Error())
 		return err
 	}
 	err = models.CreateCloudbrain(&models.Cloudbrain{
-		Status:          string(models.JobWaiting),
+		Status:          jobResult.Status,
 		UserID:          ctx.User.ID,
 		RepoID:          ctx.Repo.Repository.ID,
 		JobID:           jobResult.ID,
@@ -297,6 +306,7 @@ func GenerateNotebook2(ctx *context.Context, jobName, uuid, description, flavor
 		Type:            models.TypeCloudBrainTwo,
 		Uuid:            uuid,
 		ComputeResource: models.NPUResource,
+		Image: 			 imageName,
 	})
 
 	if err != nil {
@@ -599,3 +609,26 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e
 	notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.JobName, models.ActionCreateInferenceTask)
 	return nil
 }
+
+func GetNotebookImageName(imageId string) (string, error) {
+	var validImage = false
+	var imageName = ""
+
+	if ImageInfos == nil {
+		json.Unmarshal([]byte(setting.ImageInfos), &ImageInfos)
+	}
+
+	for _, imageInfo := range ImageInfos.ImageInfo {
+		if imageInfo.Id == imageId {
+			validImage = true
+			imageName = imageInfo.Value
+		}
+	}
+
+	if !validImage {
+		log.Error("the image id(%s) is invalid", imageId)
+		return imageName, errors.New("the image id is invalid")
+	}
+
+	return imageName, nil
+}

modules/setting/setting.go

@@ -513,6 +513,8 @@ var (
 	PoolInfos         string
 	Flavor            string
 	DebugHost  		  string
+	ImageInfos		  string
+	Capacity		  int
 	//train-job
 	ResourcePools       string
 	Engines             string
@@ -1326,7 +1328,8 @@ func NewContext() {
 	ProfileID = sec.Key("PROFILE_ID").MustString("")
 	PoolInfos = sec.Key("POOL_INFOS").MustString("")
 	Flavor = sec.Key("FLAVOR").MustString("")
-	DebugHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73")
+	ImageInfos = sec.Key("IMAGE_INFOS").MustString("")
+	Capacity = sec.Key("IMAGE_INFOS").MustInt(100)
 	ResourcePools = sec.Key("Resource_Pools").MustString("")
 	Engines = sec.Key("Engines").MustString("")
 	EngineVersions = sec.Key("Engine_Versions").MustString("")

routers/repo/modelarts.go

@@ -103,8 +103,13 @@ func MustEnableModelArts(ctx *context.Context) {
 }
 
 func NotebookNew(ctx *context.Context) {
-	ctx.Data["PageIsCloudBrain"] = true
+	notebookNewDataPrepare(ctx)
+
+	ctx.HTML(200, tplModelArtsNotebookNew)
+}
 
+func notebookNewDataPrepare(ctx *context.Context) error {
+	ctx.Data["PageIsCloudBrain"] = true
 	t := time.Now()
 	var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
 	ctx.Data["job_name"] = jobName
@@ -112,26 +117,14 @@ func NotebookNew(ctx *context.Context) {
 	attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
 	if err != nil {
 		ctx.ServerError("GetAllUserAttachments failed:", err)
-		return
+		return err
 	}
-
 	ctx.Data["attachments"] = attachs
-	ctx.Data["dataset_path"] = modelarts.DataSetMountPath
-	ctx.Data["env"] = modelarts.NotebookEnv
-	ctx.Data["notebook_type"] = modelarts.NotebookType
-	if modelarts.FlavorInfos == nil {
-		json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos)
-	}
-	ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo
 
-	ctx.HTML(200, tplModelArtsNotebookNew)
-}
-
-func notebookNewDataPrepare(ctx *context.Context) error {
-	ctx.Data["PageIsCloudBrain"] = true
-	t := time.Now()
-	var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
-	ctx.Data["job_name"] = jobName
+	if modelarts.ImageInfos == nil {
+		json.Unmarshal([]byte(setting.ImageInfos), &modelarts.ImageInfos)
+	}
+	ctx.Data["images"] = modelarts.ImageInfos.ImageInfo
 
 	if modelarts.FlavorInfos == nil {
 		json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos)
@@ -191,8 +184,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm
 	uuid := form.Attachment
 	description := form.Description
 	flavor := form.Flavor
-
-	flavor = "modelarts.bm.910.arm.public.1"
+	imageId := form.ImageId
 
 	count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID)
 	if err != nil {
@@ -223,7 +215,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm
 		}
 	}
 
-	err = modelarts.GenerateNotebook2(ctx, jobName, uuid, description, flavor)
+	err = modelarts.GenerateNotebook2(ctx, jobName, uuid, description, flavor, imageId)
 	if err != nil {
 		log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"])
 		notebookNewDataPrepare(ctx)
@@ -262,10 +254,17 @@ func NotebookShow(ctx *context.Context) {
 
 		result.CreateTime = time.Unix(int64(result.CreateAt/1000), 0).Format("2006-01-02 15:04:05")
 		result.LatestUpdateTime = time.Unix(int64(result.UpdateAt/1000), 0).Format("2006-01-02 15:04:05")
-		//result.QueuingInfo.BeginTime = time.Unix(int64(result.QueuingInfo.BeginTimestamp/1000), 0).Format("2006-01-02 15:04:05")
-		//result.QueuingInfo.EndTime = time.Unix(int64(result.QueuingInfo.EndTimestamp/1000), 0).Format("2006-01-02 15:04:05")
 	}
 
+	var datasetDownloadLink string
+	if task.Uuid != "" && task.UserID == ctx.User.ID {
+		attachment, err := models.GetAttachmentByUUID(task.Uuid)
+		if err == nil {
+			datasetDownloadLink = attachment.S3DownloadURL()
+		}
+	}
+
+	ctx.Data["datasetDownloadLink"] = datasetDownloadLink
 	ctx.Data["task"] = task
 	ctx.Data["jobID"] = jobID
 	ctx.Data["jobName"] = task.JobName

templates/repo/modelarts/notebook/new.tmpl

@@ -51,7 +51,16 @@
                         <input name="job_name" id="cloudbrain_job_name" placeholder="任务名称" value="{{.job_name}}" tabindex="3" autofocus required maxlength="255" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
                     </div>
 
-                    <!-- <div class="inline field">
+                    <div class="inline required field">
+                        <label>镜像</label>
+                        <select id="cloudbrain_image" class="ui search dropdown" placeholder="选择镜像" style='width:385px' name="image_id">
+                            {{range .images}}
+                                <option name="image_id" value="{{.Id}}">{{.Value}}</option>
+                            {{end}}
+                        </select>
+                    </div>
+
+                    <div class="inline field">
                         <label>数据集</label>
                         <input type="text" list="cloudbrain_dataset" placeholder="选择数据集" name="" id="answerInput" autofocus maxlength="36">
                         <datalist id="cloudbrain_dataset" class="ui search" style='width:385px' name="attachment">
@@ -62,7 +71,7 @@
                         <input type="hidden" name="attachment" id="answerInput-hidden">
                     </div>
 
-                    <div class="inline required field">
+                    <!--<div class="inline required field">
                         <label>工作环境</label>
                         <input name="de" id="cloudbrain_de" value="{{.env}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
                     </div>

templates/repo/modelarts/notebook/show.tmpl

@@ -27,7 +27,7 @@
             {{end}}
           </div>
           <div class="ui green segment">
-            <p>任务结果:</p>
+            <p>任务详情:</p>
             {{with .result}}
             <table class="ui celled striped table">
               <tbody>
@@ -36,6 +36,14 @@
                   <td> {{.Status}} </td>
                 </tr>
                 <tr>
+                  <td> 镜像名称 </td>
+                  <td>{{$.task.Image}}</td>
+                </tr>
+                <tr>
+                  <td> 数据集下载地址 </td>
+                  <td style="max-width: 480px; word-wrap:break-word">{{$.datasetDownloadLink}}</td>
+                </tr>
+                <tr>
                   <td> 开始时间 </td>
                   <td>{{.CreateTime}}</td>
                 </tr>