diff --git a/.gitignore b/.gitignore index 9f34fea2a..ceeeaa92c 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ coverage.all !/custom/conf/templates /custom/conf/app.ini !/custom/conf/app.ini.sample +/custom/public/kanban /data /indexers /log diff --git a/models/cloudbrain.go b/models/cloudbrain.go index af53bad32..f32ad5870 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -206,7 +206,16 @@ func (task *Cloudbrain) CorrectCreateUnix() { func (task *Cloudbrain) IsTerminal() bool { status := task.Status - return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled) || status == string(ModelArtsStopped) || status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) + return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || + status == string(ModelArtsTrainJobKilled) || status == string(ModelArtsStopped) || + status == string(JobStopped) || status == string(JobFailed) || + status == string(JobSucceeded) || status == GrampusStatusFailed || + status == GrampusStatusSucceeded || status == GrampusStatusStopped +} +func (task *Cloudbrain) IsRunning() bool { + status := task.Status + return status == string(ModelArtsTrainJobRunning) || status == string(ModelArtsRunning) || + status == string(JobRunning) || status == GrampusStatusRunning } func ConvertDurationToStr(duration int64) string { diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index 6e7921d75..b6fffca0e 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -238,6 +238,9 @@ func GetAllUserPublicRepoKPIStats(startTime time.Time, endTime time.Time) (map[s CommitLines: 0, } } + if value.Email == "1250125907@qq.com" || value.Email == "peiyongyu-34@163.com" { + log.Info("repo path=" + repository.RepoPath()) + } authors[key].Commits += value.Commits authors[key].CommitLines += value.CommitLines diff --git a/modules/auth/wechat/client.go b/modules/auth/wechat/client.go index 9ed4b543f..bb6db09d0 100644 --- a/modules/auth/wechat/client.go +++ b/modules/auth/wechat/client.go @@ -19,6 +19,7 @@ const ( ACCESS_TOKEN_PATH = "/cgi-bin/token" QR_CODE_PATH = "/cgi-bin/qrcode/create" GET_MATERIAL_PATH = "/cgi-bin/material/batchget_material" + SEND_TEMPLATE_PATH = "/cgi-bin/message/template/send" ACTION_QR_STR_SCENE = "QR_STR_SCENE" ERR_CODE_ACCESSTOKEN_EXPIRE = 42001 @@ -41,12 +42,33 @@ type QRCodeRequest struct { Action_info ActionInfo `json:"action_info"` Expire_seconds int `json:"expire_seconds"` } + type MaterialRequest struct { Type string `json:"type"` Offset int `json:"offset"` Count int `json:"count"` } +type TemplateMsgRequest struct { + ToUser string `json:"touser"` + TemplateId string `json:"template_id"` + Url string `json:"url"` + ClientMsgId string `json:"client_msg_id"` + Data interface{} `json:"data"` +} +type TemplateValue struct { + Value string `json:"value"` + Color string `json:"color"` +} + +type CloudbrainTaskData struct { + First TemplateValue `json:"first"` + Keyword1 TemplateValue `json:"keyword1"` + Keyword2 TemplateValue `json:"keyword2"` + Keyword3 TemplateValue `json:"keyword3"` + Remark TemplateValue `json:"remark"` +} + type ActionInfo struct { Scene Scene `json:"scene"` } @@ -161,3 +183,27 @@ func getErrorCodeFromResponse(r *resty.Response) int { c, _ := strconv.Atoi(fmt.Sprint(code)) return c } + +func sendTemplateMsg(req TemplateMsgRequest) (error, bool) { + client := getWechatRestyClient() + + bodyJson, _ := json.Marshal(req) + r, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetQueryParam("access_token", GetWechatAccessToken()). + SetBody(bodyJson). + Post(setting.WechatApiHost + SEND_TEMPLATE_PATH) + if err != nil { + log.Error("sendTemplateMsg,e=%v", err) + return nil, false + } + a := r.Body() + resultMap := make(map[string]interface{}, 0) + json.Unmarshal(a, &resultMap) + errcode := resultMap["errcode"] + log.Info("sendTemplateMsg,%v", r) + if errcode == fmt.Sprint(ERR_CODE_ACCESSTOKEN_EXPIRE) || errcode == fmt.Sprint(ERR_CODE_ACCESSTOKEN_INVALID) { + return nil, true + } + return nil, false +} diff --git a/modules/auth/wechat/template.go b/modules/auth/wechat/template.go new file mode 100644 index 000000000..6c19651a6 --- /dev/null +++ b/modules/auth/wechat/template.go @@ -0,0 +1,145 @@ +package wechat + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "errors" + "fmt" + "time" +) + +type JobOperateType string + +const ( + JobOperateTypeStart JobOperateType = "start" + JobOperateTypeStop JobOperateType = "stop" +) + +func GetJobOperateTypeFromCloudbrainStatus(cloudbrain *models.Cloudbrain) JobOperateType { + if cloudbrain.IsTerminal() { + return JobOperateTypeStop + } + if cloudbrain.IsRunning() { + return JobOperateTypeStart + } + return "" +} + +func SendCloudbrainStartedMsg(operateType JobOperateType, cloudbrain models.Cloudbrain) error { + defer func() { + if err := recover(); err != nil { + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC:", combinedErr) + } + }() + repo, err := models.GetRepositoryByID(cloudbrain.RepoID) + if err != nil { + log.Error("SendCloudbrainStartedMsg GetRepositoryByID error,%v", err) + } + + if setting.CloudbrainStartedTemplateId == "" { + return nil + } + + openId := models.GetUserWechatOpenId(cloudbrain.UserID) + if openId == "" { + return errors.New("Wechat openId not exist") + } + data := CloudbrainTaskData{ + First: TemplateValue{Value: getCloudbrainTemplateTitle(operateType)}, + Keyword1: TemplateValue{Value: cloudbrain.DisplayJobName}, + Keyword2: TemplateValue{Value: getJobTypeDisplayName(cloudbrain.JobType)}, + Keyword3: TemplateValue{Value: time.Unix(int64(cloudbrain.CreatedUnix), 0).Format("2006-01-02 15:04:05")}, + Remark: TemplateValue{Value: getCloudbrainTemplateRemark(operateType)}, + } + req := TemplateMsgRequest{ + ToUser: openId, + TemplateId: setting.CloudbrainStartedTemplateId, + Url: getCloudbrainTemplateUrl(cloudbrain, repo), + ClientMsgId: string(operateType) + "_" + fmt.Sprint(cloudbrain.ID), + Data: data, + } + err, retryFlag := sendTemplateMsg(req) + if retryFlag { + log.Info("retrySendCloudbrainTemplateMsg calling") + refreshAccessToken() + err, _ = sendTemplateMsg(req) + if err != nil { + log.Error("SendCloudbrainStartedMsg err. %v", err) + return err + } + return nil + } + if err != nil { + log.Error("SendCloudbrainStartedMsg err. %v", err) + return err + } + return nil +} + +func getCloudbrainTemplateUrl(cloudbrain models.Cloudbrain, repo *models.Repository) string { + url := setting.AppURL + repo.FullName() + + switch cloudbrain.JobType { + case string(models.JobTypeDebug): + if cloudbrain.ComputeResource == "CPU/GPU" { + url += "/cloudbrain/" + fmt.Sprint(cloudbrain.ID) + } else { + url += "/modelarts/notebook/" + fmt.Sprint(cloudbrain.ID) + } + case string(models.JobTypeBenchmark): + url += "/cloudbrain/benchmark/" + fmt.Sprint(cloudbrain.ID) + case string(models.JobTypeTrain): + if cloudbrain.Type == models.TypeCloudBrainOne { + url += "/cloudbrain/train-job/" + fmt.Sprint(cloudbrain.JobID) + } else if cloudbrain.Type == models.TypeCloudBrainTwo { + url += "/modelarts/train-job/" + fmt.Sprint(cloudbrain.JobID) + } else if cloudbrain.Type == models.TypeC2Net { + url += "/grampus/train-job/" + fmt.Sprint(cloudbrain.JobID) + } + case string(models.JobTypeInference): + url += "/modelarts/inference-job/" + fmt.Sprint(cloudbrain.JobID) + } + return url +} + +func getCloudbrainTemplateTitle(operateType JobOperateType) string { + var title string + switch operateType { + case JobOperateTypeStart: + title = "您好，您提交的算力资源申请已通过，任务已启动，请您关注运行情况。" + case JobOperateTypeStop: + title = "您好，您提交的任务已运行结束。" + } + + return title + +} + +func getCloudbrainTemplateRemark(operateType JobOperateType) string { + var remark string + switch operateType { + case JobOperateTypeStart: + remark = "感谢您的耐心等待。" + case JobOperateTypeStop: + remark = "点击可查看运行结果" + } + + return remark + +} + +func getJobTypeDisplayName(jobType string) string { + switch jobType { + case string(models.JobTypeDebug): + return "调试任务" + case string(models.JobTypeBenchmark): + return "评测任务" + case string(models.JobTypeTrain): + return "训练任务" + case string(models.JobTypeInference): + return "推理任务" + } + return "" +} diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go index 1a7b657d5..52d91e77a 100644 --- a/modules/git/repo_stats_custom.go +++ b/modules/git/repo_stats_custom.go @@ -62,7 +62,7 @@ func GetUserKPIStats(repoPath string, startTime time.Time, endTime time.Time) (m after := startTime.Format(time.RFC3339) until := endTime.Format(time.RFC3339) - args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--after='%s'", after), fmt.Sprintf("--until=='%s'", until)} + args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--after='%s'", after), fmt.Sprintf("--until='%s'", until)} stdout, err := NewCommand(args...).RunInDirBytes(repoPath) if err != nil { return nil, err diff --git a/modules/notification/base/notifier.go b/modules/notification/base/notifier.go index 8d6fdeb52..3bdc29d46 100644 --- a/modules/notification/base/notifier.go +++ b/modules/notification/base/notifier.go @@ -56,4 +56,6 @@ type Notifier interface { NotifySyncDeleteRef(doer *models.User, repo *models.Repository, refType, refFullName string) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) + + NotifyChangeCloudbrainStatus(cloudbrain *models.Cloudbrain, oldStatus string) } diff --git a/modules/notification/base/null.go b/modules/notification/base/null.go index 0d3489882..6a24963d2 100644 --- a/modules/notification/base/null.go +++ b/modules/notification/base/null.go @@ -158,3 +158,7 @@ func (*NullNotifier) NotifySyncDeleteRef(doer *models.User, repo *models.Reposit func (*NullNotifier) NotifyOtherTask(doer *models.User, repo *models.Repository, id string, name string, optype models.ActionType) { } + +func (*NullNotifier) NotifyChangeCloudbrainStatus(cloudbrain *models.Cloudbrain, oldStatus string) { + +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 0fd6fa471..b6d925f07 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/notification/mail" "code.gitea.io/gitea/modules/notification/ui" "code.gitea.io/gitea/modules/notification/webhook" + wechatNotifier "code.gitea.io/gitea/modules/notification/wechat" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) @@ -35,6 +36,7 @@ func NewContext() { RegisterNotifier(indexer.NewNotifier()) RegisterNotifier(webhook.NewNotifier()) RegisterNotifier(action.NewNotifier()) + RegisterNotifier(wechatNotifier.NewNotifier()) } // NotifyUploadAttachment notifies attachment upload message to notifiers @@ -269,3 +271,10 @@ func NotifySyncDeleteRef(pusher *models.User, repo *models.Repository, refType, notifier.NotifySyncDeleteRef(pusher, repo, refType, refFullName) } } + +// NotifyChangeCloudbrainStatus +func NotifyChangeCloudbrainStatus(cloudbrain *models.Cloudbrain, oldStatus string) { + for _, notifier := range notifiers { + notifier.NotifyChangeCloudbrainStatus(cloudbrain, oldStatus) + } +} diff --git a/modules/notification/wechat/wechat.go b/modules/notification/wechat/wechat.go new file mode 100644 index 000000000..f77bfe741 --- /dev/null +++ b/modules/notification/wechat/wechat.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package wechat + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth/wechat" + "code.gitea.io/gitea/modules/notification/base" + "code.gitea.io/gitea/modules/setting" +) + +type wechatNotifier struct { + base.NullNotifier +} + +var ( + _ base.Notifier = &wechatNotifier{} +) + +// NewNotifier create a new wechatNotifier notifier +func NewNotifier() base.Notifier { + return &wechatNotifier{} +} + +func (*wechatNotifier) NotifyChangeCloudbrainStatus(cloudbrain *models.Cloudbrain, oldStatus string) { + operateType := wechat.GetJobOperateTypeFromCloudbrainStatus(cloudbrain) + if operateType == "" { + return + } + switch operateType { + case wechat.JobOperateTypeStart: + if len(setting.CloudbrainStartedNotifyList) == 0 { + return + } + for _, v := range setting.CloudbrainStartedNotifyList { + if v == cloudbrain.JobType { + go wechat.SendCloudbrainStartedMsg(operateType, *cloudbrain) + return + } + } + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 6ec54fdff..f63088091 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -582,6 +582,10 @@ var ( TreePathOfAutoMsgReply string TreePathOfSubscribe string + //wechat template msg config + CloudbrainStartedTemplateId string + CloudbrainStartedNotifyList []string + //nginx proxy PROXYURL string RadarMap = struct { @@ -1432,7 +1436,7 @@ func NewContext() { WechatApiHost = sec.Key("HOST").MustString("https://api.weixin.qq.com") WechatApiTimeoutSeconds = sec.Key("TIMEOUT_SECONDS").MustInt(3) WechatAppId = sec.Key("APP_ID").MustString("wxba77b915a305a57d") - WechatAppSecret = sec.Key("APP_SECRET").MustString("e48e13f315adc32749ddc7057585f198") + WechatAppSecret = sec.Key("APP_SECRET").MustString("") WechatQRCodeExpireSeconds = sec.Key("QR_CODE_EXPIRE_SECONDS").MustInt(120) WechatAuthSwitch = sec.Key("AUTH_SWITCH").MustBool(true) UserNameOfWechatReply = sec.Key("AUTO_REPLY_USER_NAME").MustString("OpenIOSSG") @@ -1440,6 +1444,8 @@ func NewContext() { RefNameOfWechatReply = sec.Key("AUTO_REPLY_REF_NAME").MustString("master") TreePathOfAutoMsgReply = sec.Key("AUTO_REPLY_TREE_PATH").MustString("wechat/auto_reply.json") TreePathOfSubscribe = sec.Key("SUBSCRIBE_TREE_PATH").MustString("wechat/subscribe_reply.json") + CloudbrainStartedTemplateId = sec.Key("CLOUDBRAIN_STARTED_TEMPLATE_ID").MustString("") + CloudbrainStartedNotifyList = strings.Split(sec.Key("CLOUDBRAIN_STARTED_NOTIFY_LIST").MustString("DEBUG"), ",") SetRadarMapConfig() diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index 522823064..4c0cbac55 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -187,9 +187,6 @@ func GetOneLevelAllObjectUnderDirMinio(bucket string, prefixRootPath string, rel if val.Key == Prefix { continue } - // if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { - // continue - // } if strings.HasSuffix(val.Key, "/") { isDir = true fileName = val.Key[prefixLen : len(val.Key)-1] diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b641f4011..105abf006 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1007,7 +1007,7 @@ cloudbrain.time.starttime=Start run time cloudbrain.time.endtime=End run time cloudbrain.datasetdownload=Dataset download url model_manager = Model -model_noright=No right +model_noright=You have no right to do the operation. model_rename=Duplicate model name, please modify model name. date=Date @@ -1225,7 +1225,7 @@ model.manage.create_new_convert_task=Create Model Transformation Task modelconvert.manage.create_error1=A model transformation task with the same name already exists. modelconvert.manage.create_error2=Only one running model transformation task can be created. modelconvert.manage.model_not_exist=The model does not exist. -modelconvert.manage.no_operate_right=No operation permission. +modelconvert.manage.no_operate_right=You have no right to do the operation. grampus.train_job.ai_center = AI Center grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。 diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 87630d6c0..1717f5cfd 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1006,7 +1006,7 @@ datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 model_manager = 模型 -model_noright=无权限操作 +model_noright=您没有操作权限。 model_rename=模型名称重复，请修改模型名称 @@ -1237,7 +1237,7 @@ model.manage.create_new_convert_task=创建模型转换任务 modelconvert.manage.create_error1=相同的名称模型转换任务已经存在。 modelconvert.manage.create_error2=只能创建一个正在运行的模型转换任务。 modelconvert.manage.model_not_exist=选择的模型不存在。 -modelconvert.manage.no_operate_right=无操作权限。 +modelconvert.manage.no_operate_right=您没有操作权限。 grampus.train_job.ai_center=智算中心 grampus.dataset_path_rule = 训练脚本存储在/cache/code中，数据集存储在/cache/dataset中，训练输出请存储在/cache/output中以供后续下载。 diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index d291024f9..c3a803f70 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -6,6 +6,7 @@ package repo import ( + "code.gitea.io/gitea/modules/notification" "encoding/json" "net/http" "sort" @@ -74,7 +75,7 @@ func GetCloudbrainTask(ctx *context.APIContext) { log.Error("ConvertToJobResultPayload failed:", err) return } - + oldStatus := job.Status job.Status = result.JobStatus.State taskRoles := result.TaskRoles taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) @@ -86,6 +87,9 @@ func GetCloudbrainTask(ctx *context.APIContext) { if result.JobStatus.State != string(models.JobWaiting) { models.ParseAndSetDurationFromCloudBrainOne(result, job) + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) @@ -99,6 +103,7 @@ func GetCloudbrainTask(ctx *context.APIContext) { "SubState": result.JobStatus.SubState, "CreatedTime": time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"), "CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"), + "JobDuration": job.TrainJobDuration, }) } @@ -123,7 +128,7 @@ func GetCloudBrainInferenceJob(ctx *context.APIContext) { log.Error("ConvertToJobResultPayload failed:", err) return } - + oldStatus := job.Status job.Status = result.JobStatus.State if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { taskRoles := result.TaskRoles @@ -136,6 +141,9 @@ func GetCloudBrainInferenceJob(ctx *context.APIContext) { if result.JobStatus.State != string(models.JobWaiting) { models.ParseAndSetDurationFromCloudBrainOne(result, job) + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 187c16c50..7d30614b5 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -6,6 +6,7 @@ package repo import ( + "code.gitea.io/gitea/modules/notification" "encoding/json" "net/http" "path" @@ -42,8 +43,11 @@ func GetModelArtsNotebook(ctx *context.APIContext) { ctx.NotFound(err) return } - + oldStatus := job.Status job.Status = result.Status + if oldStatus != result.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) @@ -75,21 +79,26 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { if job.StartTime == 0 && result.Lease.UpdateTime > 0 { job.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) } + oldStatus := job.Status job.Status = result.Status if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) { job.EndTime = timeutil.TimeStampNow() } job.CorrectCreateUnix() job.ComputeAndSetDuration() + if oldStatus != result.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) } ctx.JSON(http.StatusOK, map[string]interface{}{ - "ID": ID, - "JobName": job.JobName, - "JobStatus": result.Status, + "ID": ID, + "JobName": job.JobName, + "JobStatus": result.Status, + "JobDuration": job.TrainJobDuration, }) } @@ -111,10 +120,13 @@ func GetModelArtsTrainJob(ctx *context.APIContext) { ctx.NotFound(err) return } - + oldStatus := job.Status job.Status = modelarts.TransTrainJobStatus(result.IntStatus) job.Duration = result.Duration job.TrainJobDuration = result.TrainJobDuration + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) @@ -155,7 +167,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { log.Error("ConvertToJobResultPayload failed:", err) return } - + oldStatus := job.Status job.Status = result.JobStatus.State if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { taskRoles := result.TaskRoles @@ -168,6 +180,9 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { if result.JobStatus.State != string(models.JobWaiting) { models.ParseAndSetDurationFromCloudBrainOne(result, job) + if oldStatus != job.Status { + notification.NotifyChangeCloudbrainStatus(job, oldStatus) + } err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 0aef1a70c..d01539a75 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -152,6 +152,10 @@ func saveModelByParameters(jobId string, versionName string, name string, versio } func SaveNewNameModel(ctx *context.Context) { + if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { + ctx.Error(403, ctx.Tr("repo.model_noright")) + return + } name := ctx.Query("Name") if name == "" { ctx.Error(500, fmt.Sprintf("name or version is null.")) @@ -169,6 +173,10 @@ func SaveNewNameModel(ctx *context.Context) { } func SaveModel(ctx *context.Context) { + if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { + ctx.Error(403, ctx.Tr("repo.model_noright")) + return + } log.Info("save model start.") JobId := ctx.Query("JobId") VersionName := ctx.Query("VersionName") @@ -177,16 +185,8 @@ func SaveModel(ctx *context.Context) { label := ctx.Query("Label") description := ctx.Query("Description") engine := ctx.QueryInt("Engine") - trainTaskCreate := ctx.QueryBool("trainTaskCreate") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) - if !trainTaskCreate { - if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { - //ctx.NotFound(ctx.Req.URL.RequestURI(), nil) - ctx.JSON(403, ctx.Tr("repo.model_noright")) - return - } - } if JobId == "" || VersionName == "" { ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 6151f0070..777dbcb10 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -15,6 +15,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/timeutil" @@ -218,6 +220,255 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { return nil } +func cloudBrainTrainJobErrorPrepare(ctx *context.Context, form auth.CreateCloudBrainForm) error { + ctx.Data["PageIsCloudBrain"] = true + + if categories == nil { + json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) + } + ctx.Data["benchmark_categories"] = categories.Category + + ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + } + + cloudbrain.InitSpecialPool() + + if gpuInfos == nil { + json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) + } + ctx.Data["gpu_types"] = gpuInfos.GpuInfo + + if trainGpuInfos == nil { + json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos) + } + ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo + + if inferenceGpuInfos == nil && setting.InferenceGpuTypes != "" { + json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos) + } + if inferenceGpuInfos != nil { + ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo + } + + if benchmarkGpuInfos == nil { + json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) + } + ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo + + if benchmarkResourceSpecs == nil { + json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs) + } + ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec + + if cloudbrain.ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) + } + ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec + + if cloudbrain.TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) + } + ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec + + if cloudbrain.InferenceResourceSpecs == nil && setting.InferenceResourceSpecs != "" { + json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs) + } + if cloudbrain.InferenceResourceSpecs != nil { + ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec + } + + if cloudbrain.SpecialPools != nil { + var debugGpuTypes []*models.GpuInfo + var trainGpuTypes []*models.GpuInfo + + for _, pool := range cloudbrain.SpecialPools.Pools { + org, _ := models.GetOrgByName(pool.Org) + if org != nil { + isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID) + if isOrgMember { + for _, jobType := range pool.JobType { + if jobType == string(models.JobTypeDebug) { + debugGpuTypes = append(debugGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["resource_specs"] = pool.ResourceSpec + } + } else if jobType == string(models.JobTypeTrain) { + trainGpuTypes = append(trainGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["train_resource_specs"] = pool.ResourceSpec + } + } + } + break + } + } + + } + + if len(debugGpuTypes) > 0 { + ctx.Data["gpu_types"] = debugGpuTypes + } + + if len(trainGpuTypes) > 0 { + ctx.Data["train_gpu_types"] = trainGpuTypes + } + + } + + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(form.Params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["params"] = Parameters.Parameter + ctx.Data["boot_file"] = form.BootFile + ctx.Data["attachment"] = form.Attachment + _, datasetNames, err := models.GetDatasetInfo(form.Attachment) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil + } + ctx.Data["dataset_name"] = datasetNames + ctx.Data["branch_name"] = form.BranchName + ctx.Data["datasetType"] = models.TypeCloudBrainOne + + ctx.Data["display_job_name"] = form.DisplayJobName + ctx.Data["image"] = form.Image + ctx.Data["job_type"] = form.JobType + ctx.Data["gpu_type"] = form.GpuType + ctx.Data["resource_spec_id"] = form.ResourceSpecId + return nil +} + +func cloudBrainInferenceJobErrorPrepare(ctx *context.Context, form auth.CreateCloudBrainInferencForm) error { + ctx.Data["PageIsCloudBrain"] = true + + if categories == nil { + json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) + } + ctx.Data["benchmark_categories"] = categories.Category + + ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType + queuesDetail, _ := cloudbrain.GetQueuesDetail() + if queuesDetail != nil { + ctx.Data["QueuesDetail"] = queuesDetail + } + + cloudbrain.InitSpecialPool() + + if gpuInfos == nil { + json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) + } + ctx.Data["gpu_types"] = gpuInfos.GpuInfo + + if trainGpuInfos == nil { + json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos) + } + ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo + + if inferenceGpuInfos == nil && setting.InferenceGpuTypes != "" { + json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos) + } + if inferenceGpuInfos != nil { + ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo + } + + if benchmarkGpuInfos == nil { + json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) + } + ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo + + if benchmarkResourceSpecs == nil { + json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs) + } + ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec + + if cloudbrain.ResourceSpecs == nil { + json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) + } + ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec + + if cloudbrain.TrainResourceSpecs == nil { + json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) + } + ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec + + if cloudbrain.InferenceResourceSpecs == nil && setting.InferenceResourceSpecs != "" { + json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs) + } + if cloudbrain.InferenceResourceSpecs != nil { + ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec + } + + if cloudbrain.SpecialPools != nil { + var debugGpuTypes []*models.GpuInfo + var trainGpuTypes []*models.GpuInfo + + for _, pool := range cloudbrain.SpecialPools.Pools { + org, _ := models.GetOrgByName(pool.Org) + if org != nil { + isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID) + if isOrgMember { + for _, jobType := range pool.JobType { + if jobType == string(models.JobTypeDebug) { + debugGpuTypes = append(debugGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["resource_specs"] = pool.ResourceSpec + } + } else if jobType == string(models.JobTypeTrain) { + trainGpuTypes = append(trainGpuTypes, pool.Pool...) + if pool.ResourceSpec != nil { + ctx.Data["train_resource_specs"] = pool.ResourceSpec + } + } + } + break + } + } + + } + if len(debugGpuTypes) > 0 { + ctx.Data["gpu_types"] = debugGpuTypes + } + + if len(trainGpuTypes) > 0 { + ctx.Data["train_gpu_types"] = trainGpuTypes + } + + } + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(form.Params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["params"] = Parameters.Parameter + ctx.Data["boot_file"] = form.BootFile + ctx.Data["attachment"] = form.Attachment + _, datasetNames, err := models.GetDatasetInfo(form.Attachment) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil + } + ctx.Data["dataset_name"] = datasetNames + ctx.Data["branch_name"] = form.BranchName + ctx.Data["datasetType"] = models.TypeCloudBrainOne + + ctx.Data["display_job_name"] = form.DisplayJobName + ctx.Data["image"] = form.Image + ctx.Data["job_type"] = form.JobType + ctx.Data["gpu_type"] = form.GpuType + ctx.Data["resource_spec_id"] = form.ResourceSpecId + ctx.Data["label_names"] = form.LabelName + ctx.Data["train_url"] = form.TrainUrl + ctx.Data["ckpt_name"] = form.CkptName + ctx.Data["model_name"] = form.ModelName + ctx.Data["model_version"] = form.ModelVersion + ctx.Data["description"] = form.Description + return nil +} func CloudBrainNew(ctx *context.Context) { err := cloudBrainNewDataPrepare(ctx) @@ -251,28 +502,28 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr("jobtype error", tpl, &form) return } @@ -280,13 +531,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -320,7 +571,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { errStr := checkCloudBrainSpecialPool(ctx, jobType, gpuQueue, resourceSpecId) if errStr != "" { - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr(errStr, tpl, &form) return } @@ -366,7 +617,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainTrainJobErrorPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -406,20 +657,21 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } @@ -427,13 +679,13 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -453,7 +705,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -490,7 +742,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainInferenceJobErrorPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -789,12 +1041,16 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) ctx.Data["taskRes"] = taskRes ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics + oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State task.ContainerID = taskRes.TaskStatuses[0].ContainerID task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) if task.DeletedAt.IsZero() { //normal record + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { ctx.Data["error"] = err.Error() @@ -1148,12 +1404,15 @@ func CloudBrainStop(ctx *context.Context) { errorMsg = "cloudbrain.Stopped_failed" break } - + oldStatus := task.Status task.Status = string(models.JobStopped) if task.EndTime == 0 { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) @@ -1247,11 +1506,15 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { if err != nil { log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err) } else { + oldStatus := taskInfo.Status taskInfo.Status = string(models.JobStopped) if taskInfo.EndTime == 0 { taskInfo.EndTime = timeutil.TimeStampNow() } taskInfo.ComputeAndSetDuration() + if oldStatus != taskInfo.Status { + notification.NotifyChangeCloudbrainStatus(taskInfo, oldStatus) + } err = models.UpdateJob(taskInfo) if err != nil { log.Warn("UpdateJob failed", err) @@ -1731,9 +1994,13 @@ func SyncCloudbrainStatus() { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) taskRoles := jobRes.TaskRoles taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + oldStatus := task.Status task.Status = taskRes.TaskStatuses[0].State if task.Status != string(models.JobWaiting) { models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) @@ -1760,6 +2027,9 @@ func SyncCloudbrainStatus() { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -1778,6 +2048,7 @@ func SyncCloudbrainStatus() { } if result != nil { + oldStatus := task.Status task.Status = result.Status if task.StartTime == 0 && result.Lease.UpdateTime > 0 { task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) @@ -1787,6 +2058,9 @@ func SyncCloudbrainStatus() { } task.CorrectCreateUnix() task.ComputeAndSetDuration() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) @@ -1801,6 +2075,7 @@ func SyncCloudbrainStatus() { } if result != nil { + oldStatus := task.Status task.Status = modelarts.TransTrainJobStatus(result.IntStatus) task.Duration = result.Duration / 1000 task.TrainJobDuration = result.TrainJobDuration @@ -1813,6 +2088,9 @@ func SyncCloudbrainStatus() { task.EndTime = task.StartTime.Add(task.Duration) } task.CorrectCreateUnix() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) @@ -1833,6 +2111,7 @@ func SyncCloudbrainStatus() { if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] } + oldStatus := task.Status task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) task.Duration = result.JobInfo.RunSec task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) @@ -1844,6 +2123,9 @@ func SyncCloudbrainStatus() { task.EndTime = task.StartTime.Add(task.Duration) } task.CorrectCreateUnix() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 7d6aa1c30..a41f884d9 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1,15 +1,8 @@ package repo import ( - "code.gitea.io/gitea/modules/auth" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/grampus" - "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" "encoding/json" "errors" - "github.com/unknwon/com" "io/ioutil" "net/http" "os" @@ -18,6 +11,15 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/auth" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/grampus" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "github.com/unknwon/com" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/cloudbrain" @@ -136,6 +138,93 @@ func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) err return nil } +func grampusTrainJobErrorPrepare(ctx *context.Context, processType string, form auth.CreateGrampusTrainJobForm) error { + ctx.Data["PageIsCloudBrain"] = true + + //get valid images + images, err := grampus.GetImages(processType) + if err != nil { + log.Error("GetImages failed:", err.Error()) + } else { + ctx.Data["images"] = images.Infos + } + + grampus.InitSpecialPool() + + ctx.Data["GPUEnabled"] = true + ctx.Data["NPUEnabled"] = true + includeCenters := make(map[string]struct{}) + excludeCenters := make(map[string]struct{}) + if grampus.SpecialPools != nil { + for _, pool := range grampus.SpecialPools.Pools { + if pool.IsExclusive { + if !IsUserInOrgPool(ctx.User.ID, pool) { + ctx.Data[pool.Type+"Enabled"] = false + } + } else { + if strings.Contains(strings.ToLower(processType), strings.ToLower(pool.Type)) { + if IsUserInOrgPool(ctx.User.ID, pool) { + for _, center := range pool.Pool { + includeCenters[center.Queue] = struct{}{} + } + } else { + for _, center := range pool.Pool { + excludeCenters[center.Queue] = struct{}{} + } + + } + + } + + } + } + } + + //get valid resource specs + specs, err := grampus.GetResourceSpecs(processType) + + grampusSpecs := getFilterSpecBySpecialPool(specs, includeCenters, excludeCenters) + + if err != nil { + log.Error("GetResourceSpecs failed:", err.Error()) + } else { + ctx.Data["flavor_infos"] = grampusSpecs + } + + if processType == grampus.ProcessorTypeGPU { + ctx.Data["datasetType"] = models.TypeCloudBrainOne + } else if processType == grampus.ProcessorTypeNPU { + ctx.Data["datasetType"] = models.TypeCloudBrainTwo + } + + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(form.Params), &Parameters); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["params"] = Parameters.Parameter + ctx.Data["boot_file"] = form.BootFile + ctx.Data["attachment"] = form.Attachment + _, datasetNames, err := models.GetDatasetInfo(form.Attachment) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil + } + ctx.Data["dataset_name"] = datasetNames + ctx.Data["branch_name"] = form.BranchName + ctx.Data["image_id"] = form.ImageID + + ctx.Data["display_job_name"] = form.DisplayJobName + ctx.Data["image"] = form.Image + ctx.Data["flavor"] = form.FlavorID + ctx.Data["flavor_name"] = form.FlavorName + ctx.Data["description"] = form.Description + ctx.Data["engine_name"] = form.EngineName + ctx.Data["work_server_number"] = form.WorkServerNumber + + return nil +} + func getFilterSpecBySpecialPool(specs *models.GetGrampusResourceSpecsResult, includeCenters map[string]struct{}, excludeCenters map[string]struct{}) []models.GrampusSpec { if len(includeCenters) == 0 && len(excludeCenters) == 0 { return specs.Infos @@ -206,14 +295,14 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain image := strings.TrimSpace(form.Image) if !jobNamePattern.MatchString(displayJobName) { - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobGPUNew, &form) return } errStr := checkSpecialPool(ctx, "GPU") if errStr != "" { - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr(errStr, tplGrampusTrainJobGPUNew, &form) return } @@ -222,13 +311,13 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.GPUResource) if err != nil { log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("system error", tplGrampusTrainJobGPUNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplGrampusTrainJobGPUNew, &form) return } @@ -237,7 +326,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //check param if err := grampusParamCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobGPUNew, &form) return } @@ -247,14 +336,14 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("the job name did already exist", tplGrampusTrainJobGPUNew, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("system error", tplGrampusTrainJobGPUNew, &form) return } @@ -264,7 +353,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain attachment, err := models.GetAttachmentByUUID(uuid) if err != nil { log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobGPUNew, &form) return } @@ -277,7 +366,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil { log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form) return } @@ -286,7 +375,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //upload code if err := uploadCodeToMinio(codeLocalPath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form) return } @@ -294,7 +383,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" if err := mkModelPath(modelPath); err != nil { log.Error("Failed to mkModelPath: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form) return } @@ -302,7 +391,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //init model readme if err := uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/"); err != nil { log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form) return } @@ -311,7 +400,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name) if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form) return } @@ -343,7 +432,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain err = grampus.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error(), ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeGPU, form) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobGPUNew, &form) return } @@ -390,14 +479,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain engineName := form.EngineName if !jobNamePattern.MatchString(displayJobName) { - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobNPUNew, &form) return } errStr := checkSpecialPool(ctx, "NPU") if errStr != "" { - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr(errStr, tplGrampusTrainJobGPUNew, &form) return } @@ -406,13 +495,13 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.NPUResource) if err != nil { log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplGrampusTrainJobNPUNew, &form) return } @@ -421,7 +510,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //check param if err := grampusParamCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form) return } @@ -431,14 +520,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("the job name did already exist", tplGrampusTrainJobNPUNew, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form) return } @@ -448,7 +537,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain attachment, err := models.GetAttachmentByUUID(uuid) if err != nil { log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobNPUNew, &form) return } @@ -461,7 +550,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil { log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("Create task failed, server timed out", tplGrampusTrainJobNPUNew, &form) return } @@ -469,14 +558,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplGrampusTrainJobNPUNew, &form) return } if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplGrampusTrainJobNPUNew, &form) return } @@ -485,7 +574,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name) if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobNPUNew, &form) return } @@ -521,7 +610,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain err = grampus.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) + grampusTrainJobErrorPrepare(ctx, grampus.ProcessorTypeNPU, form) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form) return } @@ -550,12 +639,15 @@ func GrampusStopJob(ctx *context.Context) { errorMsg = res.ErrorMsg break } - + oldStatus := task.Status task.Status = string(models.GrampusStatusStopped) if task.EndTime == 0 { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) @@ -642,6 +734,7 @@ func GrampusTrainJobShow(ctx *context.Context) { if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] } + oldStatus := task.Status task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) if task.Status != result.JobInfo.Status || result.JobInfo.Status == models.GrampusStatusRunning { task.Duration = result.JobInfo.RunSec @@ -654,6 +747,9 @@ func GrampusTrainJobShow(ctx *context.Context) { task.EndTime = task.StartTime.Add(task.Duration) } task.CorrectCreateUnix() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob failed:" + err.Error()) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index ccdc31337..e5558af7d 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -272,8 +272,10 @@ func NotebookShow(ctx *context.Context) { if result != nil { if task.DeletedAt.IsZero() { //normal record if task.Status != result.Status { + oldStatus := task.Status task.Status = result.Status models.ParseAndSetDurationFromModelArtsNotebook(result, task) + notification.NotifyChangeCloudbrainStatus(task, oldStatus) err = models.UpdateJob(task) if err != nil { log.Error("GET job error", err.Error()) @@ -510,11 +512,15 @@ func NotebookManage(ctx *context.Context) { ID = strconv.FormatInt(newTask.ID, 10) notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, ID, task.DisplayJobName, models.ActionCreateDebugNPUTask) } else { + oldStatus := task.Status task.Status = res.Status if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { task.EndTime = timeutil.TimeStampNow() } task.ComputeAndSetDuration() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) @@ -539,7 +545,7 @@ func NotebookDel(ctx *context.Context) { var listType = ctx.Query("debugListType") task := ctx.Cloudbrain - if task.Status != string(models.ModelArtsCreateFailed) && task.Status != string(models.ModelArtsStartFailed) && task.Status != string(models.ModelArtsStopped) { + if task.Status != string(models.ModelArtsCreateFailed) && task.Status != string(models.ModelArtsStartFailed) && task.Status != string(models.ModelArtsStopped) && task.Status != string(models.ModelArtsDeleted) { log.Error("the job(%s) has not been stopped", task.JobName) ctx.RenderWithErr("the job has not been stopped", tplDebugJobIndex, nil) return @@ -772,6 +778,12 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts ctx.Data["config_list"] = configList.ParaConfigs ctx.Data["bootFile"] = form.BootFile ctx.Data["uuid"] = form.Attachment + _, datasetNames, err := models.GetDatasetInfo(form.Attachment) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil + } + ctx.Data["dataset_name"] = datasetNames ctx.Data["branch_name"] = form.BranchName ctx.Data["datasetType"] = models.TypeCloudBrainTwo @@ -2280,6 +2292,12 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel ctx.Data["config_list"] = configList.ParaConfigs ctx.Data["bootFile"] = form.BootFile ctx.Data["uuid"] = form.Attachment + _, datasetNames, err := models.GetDatasetInfo(form.Attachment) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil + } + ctx.Data["dataset_name"] = datasetNames ctx.Data["branch_name"] = form.BranchName ctx.Data["model_name"] = form.ModelName ctx.Data["model_version"] = form.ModelVersion diff --git a/routers/search.go b/routers/search.go index 05074df55..ce7782236 100644 --- a/routers/search.go +++ b/routers/search.go @@ -313,9 +313,8 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending, "num_stars", false)...).From(from).Size(Size).Highlight(queryHighlight("alias", "description", "topics")).Do(ctx.Req.Context()) if err == nil { - searchJson, _ := json.Marshal(res) - log.Info("searchJson=" + string(searchJson)) esresult := makeRepoResult(res, Key, OnlyReturnNum, language) + setForkRepoOrder(esresult) resultObj.Total = resultObj.PrivateTotal + esresult.Total isNeedSort := false if len(resultObj.Result) > 0 { @@ -348,6 +347,30 @@ func searchRepo(ctx *context.Context, TableName string, Key string, Page int, Pa } } +func setForkRepoOrder(esresult *SearchRes) { + forkidMap := make(map[int64]int, 0) + for index, re := range esresult.Result { + if re["fork_id"] != nil { + fork_id := re["fork_id"].(int64) + if _, ok := forkidMap[fork_id]; !ok { + forkidMap[fork_id] = index + } + } + } + for key, value := range forkidMap { + for index, re := range esresult.Result { + if re["id"].(int64) == key { + if value < index { //swap + tmp := esresult.Result[index] + esresult.Result[index] = esresult.Result[value] + esresult.Result[value] = tmp + break + } + } + } + } +} + func sortRepo(Result []map[string]interface{}, SortBy string, ascending bool) { orderBy := "" switch SortBy { @@ -479,6 +502,7 @@ func makeRepoResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool, record["num_stars"] = recordSource["num_stars"] record["num_forks"] = recordSource["num_forks"] record["lower_alias"] = recordSource["lower_alias"] + record["fork_id"] = recordSource["fork_id"] if recordSource["topics"] != nil { topicsStr := recordSource["topics"].(string) log.Info("topicsStr=" + topicsStr) diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index e66f40e84..174ef3ce3 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -95,7 +95,7 @@ {{else if eq .JobType "INFERENCE"}} {{.DisplayJobName}} diff --git a/templates/admin/dataset/list.tmpl b/templates/admin/dataset/list.tmpl index 9e4e72b68..9712f2e7b 100644 --- a/templates/admin/dataset/list.tmpl +++ b/templates/admin/dataset/list.tmpl @@ -35,7 +35,7 @@ {{range .Datasets}} {{.ID}} - {{.Title}}{{if .Recommend}}{{end}} + {{.Title}}{{if .Recommend}}{{end}} {{.CreatedUnix.FormatShort}} {{if .Recommend}}{{$.i18n.Tr "admin.datasets.unrecommend"}}{{else}}{{$.i18n.Tr "admin.datasets.recommend"}}{{end}} @@ -48,4 +48,4 @@ {{template "base/paginate" .}} -{{template "base/footer" .}} +{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/base/head.tmpl b/templates/base/head.tmpl index 937abd588..4ab9da231 100755 --- a/templates/base/head.tmpl +++ b/templates/base/head.tmpl @@ -109,6 +109,11 @@ ], {{end}} }; + {{if .IsSigned}} + window.sessionStorage.setItem('_csrf', '{{.CsrfToken}}'); + {{else}} + window.sessionStorage.removeItem('_csrf'); + {{end}} diff --git a/templates/base/head_course.tmpl b/templates/base/head_course.tmpl index 3fff0b8d2..fe21a0e05 100644 --- a/templates/base/head_course.tmpl +++ b/templates/base/head_course.tmpl @@ -109,6 +109,11 @@ ], {{end}} }; + {{if .IsSigned}} + window.sessionStorage.setItem('_csrf', '{{.CsrfToken}}'); + {{else}} + window.sessionStorage.removeItem('_csrf'); + {{end}} diff --git a/templates/base/head_fluid.tmpl b/templates/base/head_fluid.tmpl index 5340c7cb8..1c16e7140 100644 --- a/templates/base/head_fluid.tmpl +++ b/templates/base/head_fluid.tmpl @@ -109,6 +109,11 @@ ], {{end}} }; + {{if .IsSigned}} + window.sessionStorage.setItem('_csrf', '{{.CsrfToken}}'); + {{else}} + window.sessionStorage.removeItem('_csrf'); + {{end}} diff --git a/templates/base/head_home.tmpl b/templates/base/head_home.tmpl index 2def358f8..a77d7b00a 100644 --- a/templates/base/head_home.tmpl +++ b/templates/base/head_home.tmpl @@ -109,6 +109,11 @@ ], {{end}} }; + {{if .IsSigned}} + window.sessionStorage.setItem('_csrf', '{{.CsrfToken}}'); + {{else}} + window.sessionStorage.removeItem('_csrf'); + {{end}} diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 51761a7e5..a5a898ad0 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -42,7 +42,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} @@ -73,7 +74,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index 6baeced54..84781db11 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -41,7 +41,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} @@ -71,7 +72,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index c9ea13b8a..64e04b4c3 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -33,7 +33,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} @@ -64,7 +65,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl index e744508f0..e9f662bbe 100644 --- a/templates/base/head_navbar_pro.tmpl +++ b/templates/base/head_navbar_pro.tmpl @@ -43,7 +43,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} @@ -74,7 +75,8 @@ {{.i18n.Tr "explore.organizations"}} {{.i18n.Tr "explore.images"}} {{if .IsOperator}} - {{.i18n.Tr "explore.data_analysis"}} + {{/* {{.i18n.Tr "explore.data_analysis"}} */}} + {{.i18n.Tr "explore.data_analysis"}} {{end}} {{.i18n.Tr "custom.head.openi.repo"}} diff --git a/templates/base/head_pro.tmpl b/templates/base/head_pro.tmpl index 75292b6fc..c643ea873 100644 --- a/templates/base/head_pro.tmpl +++ b/templates/base/head_pro.tmpl @@ -109,6 +109,11 @@ ], {{end}} }; + {{if .IsSigned}} + window.sessionStorage.setItem('_csrf', '{{.CsrfToken}}'); + {{else}} + window.sessionStorage.removeItem('_csrf'); + {{end}} diff --git a/templates/explore/dataset_list.tmpl b/templates/explore/dataset_list.tmpl index 1d22b62ec..a0f171195 100755 --- a/templates/explore/dataset_list.tmpl +++ b/templates/explore/dataset_list.tmpl @@ -24,9 +24,9 @@
{{range .Datasets}}
-
- - {{.Repo.OwnerName}} / {{.Repo.Alias}} +
+ + {{.Title}}
{{if .Task}} @@ -40,7 +40,7 @@
{{if .Description}} -

{{.Description}}

+

{{.Description}}

{{else if .Repo.DescriptionHTML}}

{{.Repo.DescriptionHTML}}

{{end}} @@ -53,4 +53,7 @@
{{end}}
-
\ No newline at end of file +
+ \ No newline at end of file diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl index ff4e5e7ea..add7d34d4 100755 --- a/templates/repo/cloudbrain/benchmark/show.tmpl +++ b/templates/repo/cloudbrain/benchmark/show.tmpl @@ -353,7 +353,15 @@
- {{.Image}} + + {{.Image}} +
diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl index 223fcfe1c..7d06c55ea 100644 --- a/templates/repo/cloudbrain/inference/new.tmpl +++ b/templates/repo/cloudbrain/inference/new.tmpl @@ -58,18 +58,16 @@
{{.CsrfTokenHtml}} - - - {{if $.model_version}} - - {{else}} + - {{end}} - {{if $.label_names}} - - {{else}} - - {{end}} + + + + + + + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

@@ -102,7 +100,11 @@
+ {{if .description}} + + {{else}} + {{end}}
@@ -112,13 +114,8 @@
- {{if $.ckpt_name}} - -
{{$.model_name}}
- {{else}}
- {{end}} @@ -126,13 +123,8 @@
@@ -196,8 +196,8 @@
- {{if .bootFile}} - + {{if .boot_file}} + {{else}} {{end}} @@ -213,22 +213,22 @@ {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}
- {{if ne 0 (len .params)}} - {{range $k ,$v := .params}} -
-
- -
-
- -
- - - + {{if ne 0 (len .params)}} + {{range $k ,$v := .params}} +
+
+ +
+
+ +
+ + + -
+
+ {{end}} {{end}} - {{end}}
@@ -244,9 +244,22 @@
@@ -270,27 +283,35 @@ const RepoLink = {{.RepoLink}} let nameMap,nameList // 获取模型列表和模型名称对应的模型版本 - $.get(`${RepoLink}/modelmanage/query_model_for_predict?type=0`, (data) => { + $(document).ready(function(){ modelVersion() modelCkpt() - nameMap = data.nameMap - nameList = data.nameList - let html = '' - nameList.forEach(element => { - html += `
${element}
` - }); - if(nameList.length!==0){ - const initModelVersion = nameMap[nameList[0]][0] - const initTrainTaskInfo = JSON.parse(initModelVersion.TrainTaskInfo) - $('#model_name').append(html) - $("#select_model").dropdown('set text',nameList[0]) - $("#select_model").dropdown('set value',nameList[0],nameList[0]) - } - - $('#select_model').removeClass("loading") + $.get(`${RepoLink}/modelmanage/query_model_for_predict?type=0`, (data) => { + nameMap = data.nameMap + nameList = data.nameList + let faildModelName = document.getElementById('failed_model_name').value + let html = '' + nameList.forEach(element => { + html += `
${element}
` + }); + if(nameList.length!==0){ + $('#model_name').append(html) + if(faildModelName){ + $("#select_model").dropdown('set text',faildModelName) + $("#select_model").dropdown('set value',faildModelName) + }else{ + $("#select_model").dropdown('set text',nameList[0]) + $("#select_model").dropdown('set value',nameList[0],nameList[0]) + } + } + $('#select_model').removeClass("loading") + }) }) + // 根据选中的模型名称获取相应的模型版本 function modelVersion(){ + let faildModelVersion = $('#failed_model_version').val() + let faildTrainUrl = $('#failed_train_url').val() $('#select_model').dropdown({ onChange: function(value, text, $selectedItem) { $("#select_model_version").addClass("loading") @@ -305,13 +326,20 @@ $("#select_model_version").removeClass("loading") const initVersionText = $('#model_name_version div.item:first-child').text() const initVersionValue = $('#model_name_version div.item:first-child').data('value') - $("#select_model_version").dropdown('set text',initVersionText) - $("#select_model_version").dropdown('set value',initVersionValue,initVersionText,$('#model_name_version div.item:first-child')) + + if(faildModelVersion&&faildTrainUrl){ + $("#select_model_version").dropdown('set text',faildModelVersion) + $("#select_model_version").dropdown('set value',faildTrainUrl,faildModelVersion,$('#model_name_version div.item:first-child')) + }else{ + $("#select_model_version").dropdown('set text',initVersionText) + $("#select_model_version").dropdown('set value',initVersionValue,initVersionText,$('#model_name_version div.item:first-child')) + } } }) } // 根据选中的模型版本获取相应的模型权重文件 function modelCkpt(){ + let faildCkptName = $('#failed_ckpt_name').val() $('#select_model_version').dropdown({ onChange: function(value, text, $selectedItem) { const dataID = $selectedItem[0].getAttribute("data-id") @@ -322,21 +350,23 @@ loadCheckpointList(dataID).then((res)=>{ res.forEach(element => { const ckptSuffix = element.FileName.split(".") - const loadCheckpointFile = ['ckpt','pb','h5','json','pkl','pth','t7'] + const loadCheckpointFile = ['ckpt','pb','h5','json','pkl','pth','t7','pdparams','onnx','pbtxt','keras','mlmodel','cfg','pt'] if(!element.IsDir && loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length-1])){ html += `
${element.FileName}
` } - }) $('#model_checkpoint').append(html) $("#select_model_checkpoint").removeClass("loading") const initVersionText = $('#model_checkpoint div.item:first-child').text() const initVersionValue = $('#model_checkpoint div.item:first-child').data('value') - $("#select_model_checkpoint").dropdown('set text',initVersionText) - $("#select_model_checkpoint").dropdown('set value',initVersionValue,initVersionText,$('#model_name_version div.item:first-child')) + if(faildCkptName){ + $("#select_model_checkpoint").dropdown('set text',faildCkptName) + $("#select_model_checkpoint").dropdown('set value',faildCkptName,faildCkptName,$('#model_name_version div.item:first-child')) + }else{ + $("#select_model_checkpoint").dropdown('set text',initVersionText) + $("#select_model_checkpoint").dropdown('set value',initVersionValue,initVersionText,$('#model_name_version div.item:first-child')) + } }) - - $("input#ai_model_version").val(text) $("input#ai_model_label").val(label) } @@ -397,13 +427,6 @@ msg = JSON.stringify(msg) $('#store_run_para').val(msg) } - function get_name(){ - let name1=$("#engine_name .text").text() - let name2=$("#flaver_name .text").text() - $("input#ai_engine_name").val(name1) - $("input#ai_flaver_name").val(name2) - - } function validate(){ $('.ui.form') .form({ @@ -421,7 +444,7 @@ identifier : 'display_job_name', rules: [ { - type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[^-]$/]', + type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[a-zA-Z0-9_]$/]', } ] }, @@ -472,9 +495,8 @@ document.getElementById("mask").style.display = "none" } } + validate(); $('.ui.create_train_job.green.button').click(function(e) { send_run_para() - get_name() - validate() }) diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl index 00ad25644..055e403bd 100644 --- a/templates/repo/cloudbrain/inference/show.tmpl +++ b/templates/repo/cloudbrain/inference/show.tmpl @@ -381,7 +381,15 @@
- {{.Image}} + + {{.Image}} +
diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index f0538261d..9552c80c0 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -294,6 +294,27 @@ context.value = '' $(".icon.icons").css("visibility", "hidden") } + function validate(){ + $('.ui.form').form({ + on: 'blur', + fields: { + display_job_name:{ + identifier : 'display_job_name', + rules: [ + { + type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]', + } + ] + }, + }, + onSuccess: function(){ + }, + onFailure: function(e){ + return false; + } + }) + } + validate(); form.onsubmit = function (e) { let value_task = $("input[name='display_job_name']").val() let value_image = $("input[name='image']").val() diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index e4769c65c..0c53f7fce 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -252,7 +252,9 @@ {{$.duration}} - + + +
@@ -390,7 +392,15 @@
- {{.Image}} + + {{.Image}} +
diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index c410889b2..0208398b5 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -89,8 +89,7 @@ {{.CsrfTokenHtml}} - - +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

@@ -140,11 +139,11 @@
- + {{if .description}} + + {{else}} + + {{end}}
@@ -179,13 +178,25 @@
-
@@ -195,8 +206,8 @@
- {{if .bootFile}} - {{else}} {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}
- {{if .params}} {{if ne 0 (len .params)}} {{range $k ,$v := .params}}
@@ -238,17 +248,28 @@
{{end}} {{end}} - {{end}}
-
@@ -476,16 +497,8 @@ msg = JSON.stringify(msg) $('#store_run_para').val(msg) } - function get_name() { - let name1 = $("#engine_name .text").text() - let name2 = $("#flaver_name .text").text() - $("input#ai_engine_name").val(name1) - $("input#ai_flaver_name").val(name2) - - } + validate(); $('.ui.create_train_job.green.button').click(function (e) { - get_name() - send_run_para() - validate() + send_run_para() }) \ No newline at end of file diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 20e5cb55d..e4d8ff346 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -384,7 +384,15 @@
- {{.Image}} + + {{.Image}} +
diff --git a/templates/repo/debugjob/index.tmpl b/templates/repo/debugjob/index.tmpl index 043816e9f..ee020a8ca 100755 --- a/templates/repo/debugjob/index.tmpl +++ b/templates/repo/debugjob/index.tmpl @@ -386,7 +386,7 @@ {{$.CsrfTokenHtml}} {{if .CanDel}} {{$.i18n.Tr "repo.stop"}} @@ -405,7 +405,7 @@ {{$.CsrfTokenHtml}} {{if .CanDel}} {{$.i18n.Tr "repo.delete"}} @@ -427,6 +427,7 @@ {{if .CanDebug}} {{$.i18n.Tr "repo.submit_image"}} {{else}} {{.CsrfTokenHtml}} - - + + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

@@ -123,7 +124,11 @@
+ {{if .description}} + + {{else}} + {{end}}
@@ -156,8 +161,8 @@
- {{if .bootFile}} - + {{if .boot_file}} + {{else}} {{end}} @@ -199,8 +204,21 @@
@@ -426,16 +444,8 @@ msg = JSON.stringify(msg) $('#store_run_para').val(msg) } - function get_name(){ - let name1=$("#engine_name .text").text() - let name2=$("#flavor_name .text").text() - $("input#ai_engine_name").val(name1) - $("input#ai_flavor_name").val(name2) - - } + validate(); $('.ui.create_train_job.green.button').click(function(e) { - get_name() send_run_para() - validate() }) \ No newline at end of file diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index f23942e13..56d9a0f30 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -77,6 +77,8 @@ + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

@@ -119,7 +121,11 @@
+ {{if .description}} + + {{else}} + {{end}}
@@ -150,16 +156,29 @@
- {{if .bootFile}} - + {{if .boot_file}} + {{else}} {{end}} @@ -176,14 +195,45 @@ {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}
+ {{if .params}} + {{if ne 0 (len .params)}} + {{range $k ,$v := .params}} +
+
+ +
+
+ +
+ + + + +
+ {{end}} + {{end}} + {{end}}
@@ -425,9 +475,9 @@ $("input#trainjob_work_server_num").val(val_server_num_select) } + validate(); $('.ui.create_train_job.green.button').click(function(e) { get_name() send_run_para() - validate() }) diff --git a/templates/repo/migrate.tmpl b/templates/repo/migrate.tmpl index ef667361e..68c8f16d0 100644 --- a/templates/repo/migrate.tmpl +++ b/templates/repo/migrate.tmpl @@ -11,7 +11,7 @@ {{template "base/alert" .}}
- + {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
{{.i18n.Tr "repo.migrate.migrate_items_options"}} diff --git a/templates/repo/modelarts/inferencejob/new.tmpl b/templates/repo/modelarts/inferencejob/new.tmpl index 89f4180c0..358ec8cea 100644 --- a/templates/repo/modelarts/inferencejob/new.tmpl +++ b/templates/repo/modelarts/inferencejob/new.tmpl @@ -58,16 +58,16 @@ - {{if $.model_version}} - - {{else}} - {{end}} - {{if $.label_names}} - - {{else}} - - {{end}} + + + + + + + + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

@@ -99,24 +99,22 @@
-    + + {{if .description}} + + {{else}} + {{end}}
-

{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:

- {{if $.ckpt_name}} - -
{{$.model_name}}
- {{else}}
- {{end}} @@ -124,13 +122,8 @@
@@ -357,7 +359,15 @@
- {{.Image}} + + {{.Image}} +
diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index e89482dc2..0220d194e 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -128,7 +128,11 @@
+ {{if .description}} + + {{else}} + {{end}}
@@ -170,9 +174,22 @@
@@ -220,7 +237,6 @@
- @@ -500,9 +528,9 @@ $("input#trainjob_work_server_num").val(val_server_num_select) } + validate(); $('.ui.create_train_job.green.button').click(function (e) { get_name() send_run_para() - validate() }) \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 7a727cf4c..e61fafcdd 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -270,7 +270,7 @@ {{end}} {{if .CanDel}} -
-{{template "base/footer" .}} \ No newline at end of file +{{template "base/footer" .}} diff --git a/web_src/js/components/MinioUploader.vue b/web_src/js/components/MinioUploader.vue index 167bb8c5a..e657f6f9f 100755 --- a/web_src/js/components/MinioUploader.vue +++ b/web_src/js/components/MinioUploader.vue @@ -135,10 +135,10 @@ export default { allUploadLength(len){ if(len===this.uploadFiles.length){ setTimeout(() => { - this.dropzoneUploader.removeAllFiles(true) - this.btnFlag = false - this.$emit('setcluster',this.btnFlag) - }, 2000); + this.dropzoneUploader.removeAllFiles(true) + this.btnFlag = false + this.$emit('setcluster',this.btnFlag) + }, 2000); } } }, @@ -254,7 +254,7 @@ export default { (currentChunk / chunks) * 100 ).toFixed(2)}% (${currentChunk}/${chunks})`; - this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2)); + // this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2)); loadMd5Next(); return; } @@ -265,7 +265,7 @@ export default { file.size } 用时：${(new Date().getTime() - time) / 1000} s` ); - this.updateProgress(file,100) + // this.updateProgress(file,100) spark.destroy(); // 释放缓存 file.uniqueIdentifier = md5; // 将文件md5赋值给文件唯一标识 file.cmd5 = false; // 取消计算md5状态 @@ -297,11 +297,11 @@ export default { file.chunks = ''; this.multipartUpload(file); } else { - // 失败如何处理 - let info = "上传失败" - this.allUploadLength++ - this.uploadError(file,info) - this.allUploadFiles.push({name:file.name,status:2,info:info}) + // 失败如何处理 + let info = "上传失败" + this.allUploadLength++ + this.uploadError(file,info) + this.allUploadFiles.push({name:file.name,status:2,info:info}) return; } return; @@ -321,12 +321,16 @@ export default { this.uploadError(file,info) this.allUploadLength++ this.allUploadFiles.push({name:file.name,status:1,info:info}) + return; } - } + } console.log('文件已上传完成'); + this.allUploadLength++ + this.allUploadFiles.push({name:file.name,status:0,info:'上传成功'}) + this.updateProgress(file, 100); this.progress = 100; this.status = this.dropzoneParams.data('upload-complete'); - // this.finishUpload(file); + this.finishUpload(file); } else { // 断点续传 this.multipartUpload(file); @@ -334,6 +338,10 @@ export default { } catch (error) { this.emitDropzoneFailed(file); console.log(error); + let info = "上传失败" + this.allUploadLength++ + this.uploadError(file,info) + this.allUploadFiles.push({name:file.name,status:2,info:info}) } async function addAttachment(file) { @@ -503,6 +511,7 @@ export default { console.log(error); //this.emitDropzoneFailed(file); //console.log(error); + throw error; } } @@ -538,6 +547,11 @@ export default { await uploadChunk(e); }catch(err){ console.log(err) + let info = "上传失败" + this.allUploadLength++ + this.uploadError(file,info) + this.allUploadFiles.push({name:file.name,status:2,info:info}); + return; } fileReader.abort(); diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index cb002e76a..57c730d30 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -6,7 +6,7 @@ ref="table" :data="tableData" style="min-width: 100%" - row-key="ID" + row-key="rowKey" lazy :load="load" :tree-props="{children: 'Children', hasChildren: 'hasChildren'}" @@ -106,7 +106,7 @@
创建新版本 下载 - 删除 + 删除
@@ -171,6 +171,7 @@ export default { tableData[i].EngineName = this.getEngineName(tableData[i]) tableData[i].ComputeResource = TrainTaskInfo.ComputeResource tableData[i].cName=tableData[i].Name + tableData[i].rowKey = tableData[i].ID + Math.random() tableData[i].Name='' tableData[i].VersionCount = '' tableData[i].Children = true @@ -310,18 +311,18 @@ export default { const store = this.$refs.table.store if(!this.loadNodeMap.get(row.cName)){ const parent = store.states.data - const index = parent.findIndex(child => child.ID == row.ID) + const index = parent.findIndex(child => child.rowKey == row.rowKey) this.getModelList() }else{ let {tree,treeNode,resolve} = this.loadNodeMap.get(row.cName) const keys = Object.keys(store.states.lazyTreeNodeMap); - if(keys.includes(row.ID)){ + if(keys.includes(row.rowKey)){ this.getModelList() }else{ let parentRow = store.states.data.find(child => child.cName == row.cName); - let childrenIndex = store.states.lazyTreeNodeMap[parentRow.ID].findIndex(child => child.ID == row.ID) + let childrenIndex = store.states.lazyTreeNodeMap[parentRow.rowKey].findIndex(child => child.rowKey == row.rowKey) parentRow.VersionCount = parentRow.VersionCount-1 - const parent = store.states.lazyTreeNodeMap[parentRow.ID] + const parent = store.states.lazyTreeNodeMap[parentRow.rowKey] if(parent.length===1){ this.getModelList() }else{ @@ -331,8 +332,8 @@ export default { } }, - deleteModel(id,name){ - let row={cName:name,ID:id} + deleteModel(id,name,rowKey){ + let row={cName:name,ID:id, rowKey: rowKey} let _this = this let flag=1 $('.ui.basic.modal.first') @@ -379,8 +380,8 @@ export default { } }, getModelList(){ - try { - this.$refs.table.store.states.lazyTreeNodeMap = {} + try { + this.loadNodeMap.clear(); this.$axios.get(location.href+'_api',{ params:this.params }).then((res)=>{ @@ -391,6 +392,7 @@ export default { for(let i=0;i 私有 公开 - - + + - - + + - - + +
@@ -473,6 +473,7 @@ export default { tableDataCustom: [], starCustom:[], loadingCustom:false, + refreshCustomTimer: null, currentPageStar:1, pageSizeStar:10, @@ -485,6 +486,7 @@ export default { methods: { handleClick(tab, event) { this.search = '' + this.stopImageListCustomRefresh(); if(tab.name=="first"){ this.paramsPublic.q = '' this.getImageListPublic() @@ -560,9 +562,31 @@ export default { }); this.loadingCustom = false + this.getImageListCustomRefresh() }) }, + getImageListCustomRefresh() { + this.stopImageListCustomRefresh(); + this.refreshCustomTimer = setInterval(() => { + this.tableDataCustom.forEach(item => { + if (item.status === 0) { + this.$axios.get(`/image/${item.id}`, {}).then((res) => { + const newData = res.data; + this.tableDataCustom.forEach(it => { + if (it.id === newData.id) { + it.status = newData.status; + } + }); + }) + } + }); + }, 5000); + }, + + stopImageListCustomRefresh() { + this.refreshCustomTimer && clearInterval(this.refreshCustomTimer); + }, getImageListStar(){ this.loadingStar = true this.$axios.get('/explore/images/star',{ @@ -715,8 +739,10 @@ export default { else{ this.getImageListPublic() } + }, + beforeDestroy() { + this.stopImageListCustomRefresh(); } - }; diff --git a/web_src/js/components/images/selectGrampusImages.vue b/web_src/js/components/images/selectGrampusImages.vue index 5bbc3a43e..f702c8cca 100755 --- a/web_src/js/components/images/selectGrampusImages.vue +++ b/web_src/js/components/images/selectGrampusImages.vue @@ -225,6 +225,9 @@ export default { }, }, mounted() { + if (document.getElementById("ai_image_name")) { + this.imageAddress = document.getElementById("ai_image_name").value; + } this.getImageListPublic(); if ( location.href.indexOf("benchmark") !== -1 || diff --git a/web_src/js/components/images/selectImages.vue b/web_src/js/components/images/selectImages.vue index d4eb8d613..844303420 100755 --- a/web_src/js/components/images/selectImages.vue +++ b/web_src/js/components/images/selectImages.vue @@ -498,6 +498,9 @@ export default { }, }, mounted() { + if (document.getElementById("ai_image_name")) { + this.imageAddress = document.getElementById("ai_image_name").value; + } this.getImageListPublic(); if ( location.href.indexOf("train-job") !== -1 || diff --git a/web_src/js/features/clipboard.js b/web_src/js/features/clipboard.js index bd4a664c7..34513ad27 100644 --- a/web_src/js/features/clipboard.js +++ b/web_src/js/features/clipboard.js @@ -1,23 +1,37 @@ export default async function initClipboard() { - const els = document.querySelectorAll('.clipboard'); + const els = document.querySelectorAll(".clipboard"); if (!els || !els.length) return; - const { default: ClipboardJS } = await import(/* webpackChunkName: "clipboard" */'clipboard'); + const { default: ClipboardJS } = await import( + /* webpackChunkName: "clipboard" */ "clipboard" + ); const clipboard = new ClipboardJS(els); - clipboard.on('success', (e) => { + clipboard.on("success", (e) => { e.clearSelection(); - $(`#${e.trigger.getAttribute('id')}`).popup('destroy'); - e.trigger.setAttribute('data-content', e.trigger.getAttribute('data-success')); - $(`#${e.trigger.getAttribute('id')}`).popup('show'); - e.trigger.setAttribute('data-content', e.trigger.getAttribute('data-original')); + $(`#${e.trigger.getAttribute("id")}`).popup("destroy"); + e.trigger.setAttribute( + "data-content", + e.trigger.getAttribute("data-success") + ); + $(`#${e.trigger.getAttribute("id")}`).popup("show"); + e.trigger.setAttribute( + "data-content", + e.trigger.getAttribute("data-original") + ); }); - clipboard.on('error', (e) => { - $(`#${e.trigger.getAttribute('id')}`).popup('destroy'); - e.trigger.setAttribute('data-content', e.trigger.getAttribute('data-error')); - $(`#${e.trigger.getAttribute('id')}`).popup('show'); - e.trigger.setAttribute('data-content', e.trigger.getAttribute('data-original')); + clipboard.on("error", (e) => { + $(`#${e.trigger.getAttribute("id")}`).popup("destroy"); + e.trigger.setAttribute( + "data-content", + e.trigger.getAttribute("data-error") + ); + $(`#${e.trigger.getAttribute("id")}`).popup("show"); + e.trigger.setAttribute( + "data-content", + e.trigger.getAttribute("data-original") + ); }); } diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js index 5e9833616..12e0b71a4 100644 --- a/web_src/js/features/cloudbrainShow.js +++ b/web_src/js/features/cloudbrainShow.js @@ -175,7 +175,9 @@ export default async function initCloudrainSow() { let url = `/api/v1/repos${path}?version_name=${version_name}&parentDir=${parents}`; $.get(url, (data) => { $(`#dir_list${version_name}`).empty(); - renderDir(path, data, version_name, downloadFlag, gpuFlag); + if(data.Dirs){ + renderDir(path, data, version_name, downloadFlag, gpuFlag); + } if (init === "init") { $(`input[name=model${version_name}]`).val(""); $(`input[name=modelback${version_name}]`).val(version_name); @@ -189,7 +191,7 @@ export default async function initCloudrainSow() { htmlBread += "
/
"; $(`#file_breadcrumb${version_name}`).append(htmlBread); } else { - renderBrend(path, version_name, parents, filename, init, downloadFlag); + renderBrend(path, version_name, parents, filename, init, downloadFlag,gpuFlag); } }).fail(function (err) { console.log(err, version_name); @@ -223,7 +225,8 @@ export default async function initCloudrainSow() { parents, filename, init, - downloadFlag + downloadFlag, + gpuFlag ) { if (init == "folder") { let htmlBrend = ""; @@ -234,11 +237,11 @@ export default async function initCloudrainSow() { let filename1 = $(`input[name=modelback${version_name}]`).val(); if (parents1 === "") { $(`#file_breadcrumb${version_name} .active.section`).replaceWith( - `${sectionName}` + `${sectionName}` ); } else { $(`#file_breadcrumb${version_name} .active.section`).replaceWith( - `${sectionName}` + `${sectionName}` ); } @@ -279,7 +282,7 @@ export default async function initCloudrainSow() { html += ""; html += ""; if (data.Dirs[i].IsDir) { - html += ``; + html += ``; html += "" + data.Dirs[i].FileName + diff --git a/web_src/js/index.js b/web_src/js/index.js index 53dba8a88..b6872eaa7 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -2950,13 +2950,13 @@ $(document).ready(async () => { } const $cloneAddr = $("#clone_addr"); - $cloneAddr.on("change", () => { + $cloneAddr.on("input change", () => { const $repoName = $("#alias"); const $owner = $("#ownerDropdown div.text").attr("title"); const $urlAdd = location.href.split("/")[0] + "//" + location.href.split("/")[2]; - if ($cloneAddr.val().length > 0 && $repoName.val().length === 0) { - // Only modify if repo_name input is blank + if ($cloneAddr.val().length > 0 /* && $repoName.val().length === 0 */) { + // modify when clone address change const repoValue = $cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]; $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); $.get( @@ -3873,6 +3873,10 @@ function initVueDataset() { MinioUploader, }, mounted() { + if (document.getElementById("fail_dataset_name")) { + this.dataset_name = document.getElementById("fail_dataset_name").value; + this.dataset_uuid = document.getElementById("fail_dataset_uuid").value; + } this.getTypeList(); if (!!document.getElementById("dataset-repolink-init")) { diff --git a/web_src/less/_dataset.less b/web_src/less/_dataset.less index 35a87f89f..fff632d23 100644 --- a/web_src/less/_dataset.less +++ b/web_src/less/_dataset.less @@ -140,27 +140,26 @@ border: 1px solid #ffffff; } } - } + } } .item { - border-bottom: 1px solid rgba(34,36,38,.15); + border-bottom: 1px solid rgba(34, 36, 38, 0.15); .ui.buttons { .button { box-shadow: none !important; } } - } - + .ui.grid > .row { align-items: center; } - + .title { font-size: 16px; font-weight: bold; - margin: 0 6px; + margin: 0 6px; overflow: hidden; padding-right: 15px; white-space: nowrap; @@ -195,7 +194,15 @@ .name { word-break: break-all; } - + .dataset-title-a { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + min-width: 0; + word-break: inherit !important; + margin-right: 3rem; + white-space: nowrap; + } .metas { color: #888888; font-size: 14px; @@ -222,7 +229,7 @@ } } } -.panel_creator_reponam{ +.panel_creator_reponam { display: inline-block; border-radius: 4px; padding: 4px; @@ -231,29 +238,28 @@ background-color: rgba(161, 220, 255, 0.2); color: #101010; } -.panel_dataset_name{ +.panel_dataset_name { font-size: 15px; - color: #0366D6; + color: #0366d6; text-align: center; margin-left: 1rem; } -.panel_datset_desc{ +.panel_datset_desc { white-space: nowrap; display: inline-block; overflow: hidden; width: 90%; - + text-overflow: ellipsis; } -.el-dialog__body{ - padding-top:0 +.el-dialog__body { + padding-top: 0; } -#dataset-base{ - .active{ - color: #0087f5!important; - border: 1px solid #0087f5!important; +#dataset-base { + .active { + color: #0087f5 !important; + border: 1px solid #0087f5 !important; /* margin: -1px!important; */ - background: #fff!important; - + background: #fff !important; } -} \ No newline at end of file +} diff --git a/web_src/less/openi.less b/web_src/less/openi.less index 0c30d1c92..b9622924f 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -311,6 +311,7 @@ footer .column { } .FAILED, .START_FAILED, +.DELETED, .CREATE_FAILED { display: inline-block; width: 18px;