Merge remote-tracking branch 'origin/V20211101' into zouap

models/attachment.go

@@ -379,7 +379,7 @@ func GetUnDecompressAttachments() ([]*Attachment, error) {
 
 func getUnDecompressAttachments(e Engine) ([]*Attachment, error) {
 	attachments := make([]*Attachment, 0, 10)
-	return attachments, e.Where("decompress_state = ? and dataset_id != 0 and attachment.type = ? and (name like '%.zip' or name like '%.tar.gz' or name like '%.tgz')", DecompressStateInit, TypeCloudBrainOne).Find(&attachments)
+	return attachments, e.Where("decompress_state = ? and dataset_id != 0 and (name like '%.zip' or name like '%.tar.gz' or name like '%.tgz')", DecompressStateInit).Find(&attachments)
 }
 
 func GetAllPublicAttachments() ([]*AttachmentUsername, error) {
@@ -429,7 +429,7 @@ func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
 func getModelArtsUserAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) {
 	attachments := make([]*AttachmentUsername, 0, 10)
 	if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id "+
-		"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainNotebook, userID, false).Find(&attachments); err != nil {
+		"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainTwo, userID, false).Find(&attachments); err != nil {
 		return nil, err
 	}
 	return attachments, nil

models/cloudbrain.go

@@ -70,6 +70,7 @@ type Cloudbrain struct {
 	VersionID   int64 `xorm:"INDEX DEFAULT 0"`
 	VersionName string
 	Uuid        string
+	DatasetName string
 
 	User *User       `xorm:"-"`
 	Repo *Repository `xorm:"-"`

modules/modelarts/modelarts.go

@@ -195,6 +195,12 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) error {
 		return err
 	}
 
+	attach, err := models.GetAttachmentByUUID(req.Uuid)
+	if err != nil {
+		log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error())
+		return nil
+	}
+
 	err = models.CreateCloudbrain(&models.Cloudbrain{
 		Status:      TransTrainJobStatus(jobResult.Status),
 		UserID:      ctx.User.ID,
@@ -206,6 +212,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) error {
 		VersionID:   jobResult.VersionID,
 		VersionName: jobResult.VersionName,
 		Uuid:        req.Uuid,
+		DatasetName: attach.Name,
 	})
 
 	if err != nil {

modules/modelarts/resty.go

@@ -366,14 +366,6 @@ sendjob:
 			return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
 		}
 		log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
-		if temp.ErrorCode == "ModelArts.0404" {
-			temp.ErrorCode = "0404"
-			temp.ErrorMsg = "启动文件未找到！"
-		}
-		if temp.ErrorCode == "ModelArts.0104" {
-			temp.ErrorCode = "0104"
-			temp.ErrorMsg = "运行参数错误！"
-		}
 		return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
 	}
 

options/locale/locale_en-US.ini

@@ -764,15 +764,15 @@ submit_image=Submit Image
 download=Download
 
 
-cloudbrain=cloudbrain
+cloudbrain=Cloudbrain
 cloudbrain.new=New cloudbrain
-cloudbrain.desc=cloudbrain
+cloudbrain.desc=Cloudbrain
 cloudbrain.cancel=Cancel
-cloudbrain.commit_image = submit
-clone_cnt=download
-balance = balance
-balance.total_view = total balance
-balance.available = available balance：
+cloudbrain.commit_image = Submit
+clone_cnt=Download
+balance = Balance
+balance.total_view = Total Balance
+balance.available = Available Balance：
 cloudbrain1 = cloudbrain1
 cloudbrain2 = cloudbrain2
 cloudbrain_selection = select cloudbrain
@@ -787,7 +787,7 @@ cloudbrain_status_runtime = Running Time
 
 
 modelarts.notebook=Debug Task
-modelarts.train_job=Create Task
+modelarts.train_job=Train Task
 modelarts.train_job.new_debug= New Debug Task
 modelarts.train_job.new_train=New Train Task
 modelarts.train_job.config=Configuration information

routers/repo/attachment.go

@@ -917,16 +917,22 @@ func HandleUnDecompressAttachment() {
 	}
 
 	for _, attach := range attachs {
-		err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name)
-		if err != nil {
-			log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error())
-		} else {
-			attach.DecompressState = models.DecompressStateIng
-			err = models.UpdateAttachment(attach)
+		if attach.Type == models.TypeCloudBrainOne {
+			err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name)
 			if err != nil {
-				log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error())
+				log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error())
+			} else {
+				attach.DecompressState = models.DecompressStateIng
+				err = models.UpdateAttachment(attach)
+				if err != nil {
+					log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error())
+				}
 			}
+		} else if attach.Type == models.TypeCloudBrainTwo {
+			attachjson, _ := json.Marshal(attach)
+			labelmsg.SendDecompressAttachToLabelOBS(string(attachjson))
 		}
+
 	}
 
 	return

routers/repo/modelarts.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -35,10 +36,10 @@ const (
 	tplModelArtsNew   base.TplName = "repo/modelarts/new"
 	tplModelArtsShow  base.TplName = "repo/modelarts/show"
 
-	tplModelArtsTrainJobIndex     base.TplName = "repo/modelarts/trainjob/index"
-	tplModelArtsTrainJobNew       base.TplName = "repo/modelarts/trainjob/new"
-	tplModelArtsTrainJobShow      base.TplName = "repo/modelarts/trainjob/show"
-	tplModelArtsTrainJobShowModels		  base.TplName = "repo/modelarts/trainjob/models/index"
+	tplModelArtsTrainJobIndex      base.TplName = "repo/modelarts/trainjob/index"
+	tplModelArtsTrainJobNew        base.TplName = "repo/modelarts/trainjob/new"
+	tplModelArtsTrainJobShow       base.TplName = "repo/modelarts/trainjob/show"
+	tplModelArtsTrainJobShowModels base.TplName = "repo/modelarts/trainjob/models/index"
 )
 
 // MustEnableDataset check if repository enable internal cb
@@ -621,7 +622,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
 	logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath
-	dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
+	dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
 
 	//can, err := canUserCreateTrainJob(ctx.User.ID)
 	//if err != nil {
@@ -643,10 +644,29 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		return
 	}
 
+	attach, err := models.GetAttachmentByUUID(uuid)
+	if err != nil {
+		log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error())
+		return
+	}
+
+	//todo: del the codeLocalPath
+	_, err = ioutil.ReadDir(codeLocalPath)
+	if err == nil {
+		os.RemoveAll(codeLocalPath)
+	}
 	if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{}); err != nil {
-		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
+		log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err)
 		trainJobNewDataPrepare(ctx)
-		ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form)
+
+		ctx.Data["bootFile"] = form.BootFile
+		ctx.Data["uuid"] = form.Attachment
+		ctx.Data["datasetName"] = attach.Name
+		ctx.Data["params"] = form.Params
+		trainJobNewDataPrepare(ctx)
+		// ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form)
+		ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobNew, &form)
+		// ctx.RenderWithErr(err, tplModelArtsTrainJobNew, &form)
 		return
 	}
 
@@ -752,10 +772,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		Parameters:       param,
 	}
 
-	err := modelarts.GenerateTrainJob(ctx, req)
+	err = modelarts.GenerateTrainJob(ctx, req)
 	if err != nil {
 		log.Error("GenerateTrainJob failed:%v", err.Error())
 		trainJobNewDataPrepare(ctx)
+		ctx.Data["bootFile"] = form.BootFile
+		ctx.Data["uuid"] = form.Attachment
+		ctx.Data["datasetName"] = attach.Name
+		ctx.Data["params"] = form.Params
 		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
 		return
 	}
@@ -860,12 +884,12 @@ func TrainJobShow(ctx *context.Context) {
 		return
 	}
 
-	attach, err := models.GetAttachmentByUUID(task.Uuid)
-	if err != nil {
-		log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error())
-		ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
-		return
-	}
+	// attach, err := models.GetAttachmentByUUID(task.Uuid)
+	// if err != nil {
+	// 	log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error())
+	// 	ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
+	// 	return
+	// }
 
 	result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10))
 	if err != nil {
@@ -889,7 +913,7 @@ func TrainJobShow(ctx *context.Context) {
 			return
 		}
 
-		result.DatasetName = attach.Name
+		result.DatasetName = task.DatasetName
 	}
 
 	resultLogFile, resultLog, err := trainJobGetLog(jobID)

templates/repo/modelarts/trainjob/index.tmpl

@@ -355,7 +355,7 @@
                                         <form id="stopForm-{{.JobID}}" action="{{$.Link}}/{{.JobID}}/stop" method="post" style="margin-left:-1px;">
                                             {{$.CsrfTokenHtml}}
                                             {{if $.Permission.CanWrite $.UnitTypeCloudBrain}}
-                                            <a id="stop-model-debug-{{.JobID}}" class="ui basic {{if or (eq .Status "KILLED") (eq .Status "FAILED") (eq .Status "START_FAILED")}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
+                                            <a id="stop-model-debug-{{.JobID}}" class="ui basic {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING"}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
                                                 {{$.i18n.Tr "repo.stop"}}
                                             </a>
                                             {{else}}
@@ -535,7 +535,7 @@
                    $('#model-delete-'+jobID).removeClass('red')
                    $('#model-delete-'+jobID).addClass('disabled')
                 }
-                if(status=="KILLED" || status=="FAILED"){
+                if(status=="KILLED" || status=="FAILED" || status=="KILLING"){
                    $('#stop-model-debug-'+jobID).removeClass('blue')
                    $('#stop-model-debug-'+jobID).addClass('disabled')
                    $('#model-delete-'+jobID).removeClass('disabled')

templates/repo/modelarts/trainjob/new.tmpl

@@ -156,16 +156,12 @@
                 <h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
                 <div class="required unite min_title inline field">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
-                    <input style="width: 80%;" name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
+                    <input style="width: 60%;" name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
                 </div>
-                <!--<div class="inline field">
-                    <label>{{.i18n.Tr "repo.modelarts.train_job.version"}}</label>
-                    <span>第一版本</span>
-                </div>   
-                -->
+                
                 <div class="unite min_title inline field">
                     <label for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
-                    <textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="256" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 256)" onkeydown="this.value=this.value.substring(0, 256)" onkeyup="this.value=this.value.substring(0, 256)"></textarea>
+                    <textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 256)"></textarea>
                 </div>
                 <!-- <h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}</h4>
                 <div class="inline field">
@@ -182,7 +178,7 @@
                 <div class="required unite min_title inline fields" style="width: 90%;">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</label>
                     <div class="field" style="flex: 1.5;">
-                        <select class="ui search dropdown width" id="trainjob_engines" >
+                        <select class="ui dropdown width" id="trainjob_engines" >
                             {{range .engines}}
                                 <option value="{{.Value}}">{{.Value}}</option>
                             {{end}}
@@ -190,7 +186,7 @@
 
                     </div>
                     <div class="field" style="flex: 2;">
-                        <select class="ui search dropdown width" id="trainjob_engine_versions" style='width: 100%;' name="engine_id">
+                        <select class="ui dropdown width" id="trainjob_engine_versions" style='width: 100%;' name="engine_id">
                                 {{range .engine_versions}}
                                     <option name="engine_id" value="{{.ID}}">{{.Value}}</option>
                                 {{end}}
@@ -235,20 +231,28 @@
                 </div> -->
                 <div class="inline unite min_title field required">
                         <label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
-                        <input style="width: 33.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" >
+                        {{if .bootFile}}
+                            <input style="width: 33.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}" tabindex="3" autofocus required maxlength="255" >
+                        {{else}}
+                            <input style="width: 33.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" >
+                        {{end}}
                         <span>
                             <i class="question circle icon  link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
                         </span>
                 </div>
                 <div class="required unite min_title inline field">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
-                    <select class="ui search dropdown width80" id="trainjob_datasets" name="attachment">
+                    <select class="ui dropdown width80" id="trainjob_datasets" name="attachment" placeholder="选择数据集">
+                        {{if $.uuid}}
+                            <option name="attachment" value="{{$.uuid}}">{{$.datasetName}}</option>
+                        {{end}}
                         {{range .attachments}}
+                            <option value="">选择数据集</option>
                             <option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
                         {{end}}
                     </select>
                 </div>
-    
+                
                 <div class="inline unite min_title field">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
                     <!-- <i class="plus square outline icon"></i> -->
@@ -263,7 +267,7 @@
                 <!-- <h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.resource_setting"}}</h4> -->
                 <div class="required field " style="display: none;">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
-                    <select class="ui search dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
+                    <select class="ui dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
                         {{range .resource_pools}}
                             <option value="{{.ID}}">{{.Value}}</option>
                         {{end}}
@@ -288,7 +292,7 @@
 
                 <div class="required unite min_title inline field">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
-                    <select class="ui search dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
+                    <select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
                         {{range .flavor_infos}}
                             <option name="flavor" value="{{.Code}}">{{.Value}}</option>
                         {{end}}
@@ -297,11 +301,11 @@
                 <div class="inline required unite min_title field">
                     <label>{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
                     
-                    <div class="ui labeled input">
-                            <span class="min"><i class="minus icon"></i></span>
-                            <input style="border-radius: 0;" name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255" value="1">
+                    <div class="ui labeled input" style="width: 5%;">
+                            <!-- <span class="min"><i class="minus icon"></i></span> -->
+                            <input style="border-radius: 0;text-align: center;" name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255" value="1" disabled>
     
-                            <span class="add"><i class="plus icon"></i></span>
+                            <!-- <span class="add"><i class="plus icon"></i></span> -->
                     </div>
                     <!-- <input name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255"> -->
                 </div>
@@ -356,6 +360,9 @@
     let sever_num = $('#trainjob_work_server_num')
     $('.add').click(function(){
         sever_num.val(parseInt(sever_num.val())+1)
+        if(sever_num.val()>=26){
+            sever_num.val(parseInt(sever_num.val())-1)
+        }
     })
     $('.min').click(function(){
         sever_num.val(parseInt(sever_num.val())-1)
@@ -368,10 +375,10 @@
     function Add_parameter(i){
         value = '<div class="two fields width85" id= "para'+ i +'">' +
                     '<div class="field">' + 
-                        '<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' + 
+                        '<input type="text" name="shipping_first-name" required  placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' + 
                     '</div> ' + 
                     '<div class="field"> ' +
-                        '<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
+                        '<input type="text" name="shipping_last-name" required placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
                     '</div>'+
                     '<span>' +
                         '<i class="trash icon">' +
@@ -471,6 +478,55 @@
         $('.ui.parameter.modal')
             .modal('hide');
     })
+    $('select.dropdown')
+        .dropdown();
+
+    $('.ui.form')
+        .form({
+            on: 'blur',
+            inline:true,
+            fields: {
+                boot_file: {
+                    identifier  : 'boot_file',
+                    rules: [
+                    {
+                        type: 'regExp[/.+\.py$/g]',
+                        prompt : '启动文件必须为.py结尾'
+                    }
+                    ]
+                },
+                job_name:{
+                    identifier  : 'job_name',
+                    rules: [
+                    {
+                        type: 'regExp[/^[a-zA-Z0-9-_]{1,36}$/]',
+                        prompt : '只包含大小写字母、数字、_和-，最长36个字符。'
+                    }
+                    ]
+                },
+                attachment:{
+                    identifier  : 'attachment',
+                    rules: [
+                    {
+                        type: 'empty',
+                        prompt : '选择一个数据集'
+                    }
+                    ]
+
+                },
+                work_server_number: {
+                    identifier  : 'work_server_number',
+                    rules: [
+                    {
+                        type   : 'integer[1..25]',
+                        prompt : '计算节点需要在1-25之间，请您键入正确的值'
+                    }
+                    ]
+                }
+            },
+        })
+
+
 
     function validate(){
         $('.ui.form')
@@ -487,6 +543,25 @@
                     }
                     ]
                 },
+                job_name:{
+                    identifier  : 'job_name',
+                    rules: [
+                    {
+                        type: 'regExp[/^[a-zA-Z0-9-_]{1,36}$/]',
+                        prompt : '只包含大小写字母、数字、_和-，最长36个字符。'
+                    }
+                    ]
+                },
+                attachment:{
+                    identifier  : 'attachment',
+                    rules: [
+                    {
+                        type: 'empty',
+                        prompt : '选择一个数据集'
+                    }
+                    ]
+
+                },
                 work_server_number: {
                     identifier  : 'work_server_number',
                     rules: [