diff --git a/models/action.go b/models/action.go index db3114c95..d0a763e08 100755 --- a/models/action.go +++ b/models/action.go @@ -61,13 +61,10 @@ const ( ActionCreateGrampusNPUTrainTask //32 ActionCreateGrampusGPUTrainTask //33 ActionBindWechat //34 - ActionCreateCloudbrainTask //35 - ActionDatasetRecommended //36 - ActionCreateImage //37 - ActionImageRecommend //38 - ActionChangeUserAvatar //39 - ActionPushCommits //40 - ActionForkRepo //41 + ActionDatasetRecommended //35 + ActionCreateImage //36 + ActionImageRecommend //37 + ActionChangeUserAvatar //38 ) @@ -95,6 +92,7 @@ type Action struct { type ActionShow struct { OpType ActionType + TaskType TaskType RepoLink string ShortRepoFullDisplayName string Content string @@ -241,7 +239,8 @@ func (a *Action) GetRepoLink() string { func (a *Action) ToShow() *ActionShow { actionShow := &ActionShow{} - actionShow.OpType = GetTaskOptType(*a) + actionShow.OpType = a.OpType + actionShow.TaskType = GetTaskTypeFromAction(a.OpType) actionShow.Content = a.Content actionShow.RefName = a.RefName @@ -272,14 +271,6 @@ func (a *Action) ToShow() *ActionShow { return actionShow } -func GetTaskOptType(action Action) ActionType { - //Convert all types of cloudbrain tasks action into ActionCreateCloudbrainTask - if action.IsCloudbrainAction() { - return ActionCreateCloudbrainTask - } - return action.OpType -} - // GetRepositoryFromMatch returns a *Repository from a username and repo strings func GetRepositoryFromMatch(ownerName string, repoName string) (*Repository, error) { var err error diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c05ba0f21..bcaea544a 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -203,16 +203,17 @@ type Cloudbrain struct { } type CloudbrainShow struct { - ID int64 - JobID string - RepoFullName string - Type int - JobType string - DisplayJobName string - Duration string - ResourceSpec *Specification - ComputeResource string - AiCenter string + ID int64 + JobID string + RepoFullName string + Type int + JobType string + DisplayJobName string + Duration string + ResourceSpec *Specification + ComputeResource string + AiCenter string + WorkServerNumber int } type CloudbrainShow4Action struct { @@ -225,15 +226,20 @@ type CloudbrainShow4Action struct { } func (task *Cloudbrain) ToShow() *CloudbrainShow { + n := 1 + if task.WorkServerNumber > 1 { + n = task.WorkServerNumber + } c := &CloudbrainShow{ - ID: task.ID, - JobID: task.JobID, - JobType: task.JobType, - Type: task.Type, - DisplayJobName: task.DisplayJobName, - Duration: task.TrainJobDuration, - ResourceSpec: task.Spec, - ComputeResource: task.ComputeResource, + ID: task.ID, + JobID: task.JobID, + JobType: task.JobType, + Type: task.Type, + DisplayJobName: task.DisplayJobName, + Duration: task.TrainJobDuration, + ResourceSpec: task.Spec, + ComputeResource: task.ComputeResource, + WorkServerNumber: n, } if task.Repo != nil { c.RepoFullName = task.Repo.FullName() @@ -657,8 +663,6 @@ type FlavorInfo struct { UnitPrice int64 `json:"unitPrice"` } - - type SpecialPools struct { Pools []*SpecialPool `json:"pools"` } @@ -1476,14 +1480,23 @@ type GrampusStopJobResponse struct { } type GrampusTasks struct { - Command string `json:"command"` - Name string `json:"name"` - ImageId string `json:"imageId"` - ResourceSpecId string `json:"resourceSpecId"` - ImageUrl string `json:"imageUrl"` - CenterID []string `json:"centerID"` - CenterName []string `json:"centerName"` - ReplicaNum int `json:"replicaNum"` + Command string `json:"command"` + Name string `json:"name"` + ImageId string `json:"imageId"` + ResourceSpecId string `json:"resourceSpecId"` + ImageUrl string `json:"imageUrl"` + CenterID []string `json:"centerID"` + CenterName []string `json:"centerName"` + ReplicaNum int `json:"replicaNum"` + Datasets []GrampusDataset `json:"datasets"` + Models []GrampusDataset `json:"models"` +} + +type GrampusDataset struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + EndPoint string `json:"endPoint"` + ObjectKey string `json:"objectKey"` } type CreateGrampusJobRequest struct { diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index a14e3f886..c32e4b0fd 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -123,10 +123,14 @@ func GetResourceSpecMapByCloudbrainIDs(ids []int64) (map[int64]*Specification, e return r, nil } -func GetCloudbrainTaskUnitPrice(cloudbrainId int64) (int, error) { - s, err := GetCloudbrainSpecByID(cloudbrainId) +func GetCloudbrainTaskUnitPrice(task Cloudbrain) (int, error) { + s, err := GetCloudbrainSpecByID(task.ID) if err != nil { return 0, err } - return s.UnitPrice, nil + var n = 1 + if task.WorkServerNumber > 1 { + n = task.WorkServerNumber + } + return s.UnitPrice * n, nil } diff --git a/models/repo_watch.go b/models/repo_watch.go index 485874301..573a2d78a 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -183,6 +183,7 @@ func notifyWatchers(e Engine, actions ...*Action) error { var permCode []bool var permIssue []bool var permPR []bool + var permDataset []bool for _, act := range actions { repoChanged := repo == nil || repo.ID != act.RepoID @@ -234,12 +235,14 @@ func notifyWatchers(e Engine, actions ...*Action) error { permCode = make([]bool, len(watchers)) permIssue = make([]bool, len(watchers)) permPR = make([]bool, len(watchers)) + permDataset = make([]bool, len(watchers)) for i, watcher := range watchers { user, err := getUserByID(e, watcher.UserID) if err != nil { permCode[i] = false permIssue[i] = false permPR[i] = false + permDataset[i] = false continue } perm, err := getUserRepoPermission(e, repo, user) @@ -247,11 +250,13 @@ func notifyWatchers(e Engine, actions ...*Action) error { permCode[i] = false permIssue[i] = false permPR[i] = false + permDataset[i] = false continue } permCode[i] = perm.CanRead(UnitTypeCode) permIssue[i] = perm.CanRead(UnitTypeIssues) permPR[i] = perm.CanRead(UnitTypePullRequests) + permDataset[i] = perm.CanRead(UnitTypeDatasets) } } @@ -276,6 +281,10 @@ func notifyWatchers(e Engine, actions ...*Action) error { if !permPR[i] { continue } + case ActionDatasetRecommended: + if !permDataset[i] { + continue + } } if _, err = e.InsertOne(act); err != nil { diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index ad00cc05c..da1b6a589 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -1,8 +1,8 @@ package models import ( + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" - "fmt" "strconv" "strings" "xorm.io/builder" @@ -355,7 +355,7 @@ type RewardRecordListOpts struct { OperateType RewardOperateType RewardType RewardType SourceType string - ActionType int + TaskType string SerialNo string OrderBy RewardOperateOrderBy IsAdmin bool @@ -381,8 +381,8 @@ func (opts *RewardRecordListOpts) toCond() builder.Cond { if opts.SourceType != "" { cond = cond.And(builder.Eq{"reward_operate_record.source_type": opts.SourceType}) } - if opts.ActionType > 0 { - cond = cond.And(builder.Eq{"reward_operate_record.source_template_id": fmt.Sprint(opts.ActionType)}) + if opts.TaskType != "" { + cond = cond.And(builder.Eq{"reward_operate_record.source_template_id": opts.TaskType}) } if opts.SerialNo != "" { cond = cond.And(builder.Like{"reward_operate_record.serial_no", opts.SerialNo}) @@ -457,3 +457,22 @@ func GetAdminRewardRecordShowList(opts *RewardRecordListOpts) (RewardRecordShowL RewardRecordShowList(r).loadAttribute(true) return r, count, nil } + +func IsWechatOpenIdRewarded(wechatOpenId string) bool { + actions := make([]Action, 0) + err := x.Where(" op_type = ? and content = ?", ActionBindWechat, wechatOpenId).Find(&actions) + + if err != nil { + log.Error("IsWechatOpenIdRewarded find actions err.%v", err) + return true + } + if len(actions) == 0 { + return false + } + actionIds := make([]int64, len(actions)) + for i, v := range actions { + actionIds[i] = v.ID + } + n, _ := x.Where(builder.Eq{"source_type": SourceTypeAccomplishTask}.And(builder.In("source_id", actionIds))).Count(&RewardOperateRecord{}) + return n > 0 +} diff --git a/models/task_config.go b/models/task_config.go index baa8c9adb..0d9d21187 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -2,7 +2,6 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "fmt" "xorm.io/builder" ) @@ -11,6 +10,66 @@ const ( PeriodDaily = "DAILY" ) +type TaskType string + +const ( + TaskCreatePublicRepo TaskType = "CreatePublicRepo" + TaskCreateIssue TaskType = "CreateIssue" + TaskCreatePullRequest TaskType = "CreatePullRequest" + TaskCommentIssue TaskType = "CommentIssue" + TaskUploadAttachment TaskType = "UploadAttachment" + TaskCreateNewModelTask TaskType = "CreateNewModelTask" + TaskBindWechat TaskType = "BindWechat" + TaskCreateCloudbrainTask TaskType = "CreateCloudbrainTask" + TaskDatasetRecommended TaskType = "DatasetRecommended" + TaskCreateImage TaskType = "CreateImage" + TaskImageRecommend TaskType = "ImageRecommend" + TaskChangeUserAvatar TaskType = "ChangeUserAvatar" + TaskPushCommits TaskType = "PushCommits" +) + +func GetTaskTypeFromAction(a ActionType) TaskType { + switch a { + case ActionCreateDebugGPUTask, + ActionCreateDebugNPUTask, + ActionCreateTrainTask, + ActionCreateInferenceTask, + ActionCreateBenchMarkTask, + ActionCreateGPUTrainTask, + ActionCreateGrampusNPUTrainTask, + ActionCreateGrampusGPUTrainTask: + return TaskCreateCloudbrainTask + case ActionCreateRepo: + return TaskCreatePublicRepo + case ActionCreatePullRequest: + return TaskCreatePullRequest + case ActionCommentIssue: + return TaskCommentIssue + case ActionUploadAttachment: + return TaskUploadAttachment + case ActionCreateNewModelTask: + return TaskCreateNewModelTask + case ActionBindWechat: + return TaskBindWechat + case ActionDatasetRecommended: + return TaskDatasetRecommended + case ActionImageRecommend: + return TaskImageRecommend + case ActionCreateImage: + return TaskCreateImage + case ActionChangeUserAvatar: + return TaskChangeUserAvatar + case ActionCommitRepo, + ActionDeleteBranch, + ActionPushTag, + ActionDeleteTag: + return TaskPushCommits + case ActionCreateIssue: + return TaskCreateIssue + } + return "" +} + //PointTaskConfig Only add and delete are allowed, edit is not allowed //so if you want to edit config for some task code,please delete first and add new one type TaskConfig struct { @@ -124,8 +183,8 @@ func GetTaskConfigList() ([]*TaskConfig, error) { type GetTaskConfigOpts struct { ListOptions - Status int //1 normal 2 deleted - ActionType int + Status int //1 normal 2 deleted + TaskType string } func GetTaskConfigPageWithDeleted(opt GetTaskConfigOpts) ([]*TaskAndLimiterConfig, int64, error) { @@ -133,8 +192,8 @@ func GetTaskConfigPageWithDeleted(opt GetTaskConfigOpts) ([]*TaskAndLimiterConfi opt.Page = 1 } cond := builder.NewCond() - if opt.ActionType > 0 { - cond = cond.And(builder.Eq{"task_code": fmt.Sprint(opt.ActionType)}) + if opt.TaskType != "" { + cond = cond.And(builder.Eq{"task_code": opt.TaskType}) } var count int64 diff --git a/models/wechat_bind.go b/models/wechat_bind.go index ea005e0a6..99a38c5c3 100644 --- a/models/wechat_bind.go +++ b/models/wechat_bind.go @@ -98,5 +98,5 @@ func UnbindWechatOpenId(userId int64, oldWechatOpenID string) error { } func CountWechatBindLog(wechatOpenId string, action WechatBindAction) (int64, error) { - return x.Where("wechat_open_id = ? and action = ?", action, wechatOpenId).Count(&WechatBindLog{}) + return x.Where("wechat_open_id = ? and action = ?", wechatOpenId, action).Count(&WechatBindLog{}) } diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index ced5ea1e8..9744bc387 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -33,6 +33,7 @@ type CreateModelArtsTrainJobForm struct { DisplayJobName string `form:"display_job_name" binding:"Required"` JobName string `form:"job_name" binding:"Required"` Attachment string `form:"attachment" binding:"Required"` + DatasetName string `form:"dataset_name"` BootFile string `form:"boot_file" binding:"Required"` WorkServerNumber int `form:"work_server_number" binding:"Required"` EngineID int `form:"engine_id" binding:"Required"` diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index 83fc3b1d4..d72a7b10e 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -75,12 +75,46 @@ type GenerateTrainJobReq struct { Spec *models.Specification } +func getEndPoint() string { + index := strings.Index(setting.Endpoint, "//") + endpoint := setting.Endpoint[index+2:] + return endpoint +} + +func getDatasetGrampus(datasetInfos map[string]models.DatasetInfo) []models.GrampusDataset { + var datasetGrampus []models.GrampusDataset + endPoint := getEndPoint() + for _, datasetInfo := range datasetInfos { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: datasetInfo.FullName, + Bucket: setting.Bucket, + EndPoint: endPoint, + ObjectKey: datasetInfo.DataLocalPath + datasetInfo.FullName, + }) + + } + return datasetGrampus +} + func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { createTime := timeutil.TimeStampNow() centerID, centerName := getCentersParamter(ctx, req) - log.Info("grampus Command:" + req.Command) + var datasetGrampus, modelGrampus []models.GrampusDataset + if ProcessorTypeNPU == req.ProcessType { + datasetGrampus = getDatasetGrampus(req.DatasetInfos) + if len(req.ModelName) != 0 { + modelGrampus = []models.GrampusDataset{ + { + Name: req.ModelName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.PreTrainModelPath, + }, + } + } + } jobResult, err := createJob(models.CreateGrampusJobRequest{ Name: req.JobName, @@ -94,6 +128,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error CenterID: centerID, CenterName: centerName, ReplicaNum: 1, + Datasets: datasetGrampus, + Models: modelGrampus, }, }, }) diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index bfe574328..b1e0ec018 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -363,22 +363,15 @@ func (t *actionNotifier) NotifyWechatBind(user *models.User, wechatOpenId string func (t *actionNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { switch action { case "recommend": - users, err := models.GetAllDatasetContributorByDatasetId(dataset.ID) - if err != nil { - return - } - var actions = make([]*models.Action, 0) - for _, user := range users { - actions = append(actions, &models.Action{ - OpType: models.ActionDatasetRecommended, - ActUserID: user.ID, - ActUser: user, - RepoID: dataset.RepoID, - Repo: dataset.Repo, - Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), - }) + act := &models.Action{ + OpType: models.ActionDatasetRecommended, + ActUserID: dataset.UserID, + RepoID: dataset.RepoID, + IsPrivate: false, + Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), } - if err := models.NotifyWatchers(actions...); err != nil { + + if err := models.NotifyWatchers(act); err != nil { log.Error("notifyWatchers: %v", err) } } @@ -428,19 +421,3 @@ func (t *actionNotifier) NotifyChangeUserAvatar(user *models.User, form auth.Ava log.Error("notifyWatchers: %v", err) } } - -func (t *actionNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { - act := &models.Action{ - ActUserID: pusher.ID, - ActUser: pusher, - OpType: models.ActionPushCommits, - RepoID: repo.ID, - Repo: repo, - RefName: refName, - IsPrivate: repo.IsPrivate, - Content: fmt.Sprintf("%s|%s", oldCommitID, newCommitID), - } - if err := models.NotifyWatchers(act); err != nil { - log.Error("notifyWatchers: %v", err) - } -} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 81fc5cc06..34973abcd 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3083,6 +3083,11 @@ task_createmodel=`created new model %s` task_c2netnputrainjob=`created NPU training task %s` task_c2netgputrainjob=`created CPU/GPU training task %s` +binded_wechat=binded WeChat +dataset_recommended=`created dataset %s was set as recommended dataset` +create_image=`committed image %s` +image_recommended=`committed image %s was set as recommended image` +update_user_avatar=updated avatar [tool] ago = %s ago @@ -3202,7 +3207,7 @@ wrong_specification=You cannot use this specification, please choose another ite resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). -train_dataset_path_rule = The dataset location is stored in the environment variable data_url, and the output path is stored in the environment variable train_url. +train_dataset_path_rule = The dataset location is stored in the environment variable data_url, the pre-trained model is storaged in the environment ckpt_url, and the output path is stored in the environment variable train_url. infer_dataset_path_rule = The dataset location is stored in the environment variable data_url, and the output path is stored in the environment variable result_url. view_sample = View sample inference_output_path_rule = The inference output path is stored in the environment variable result_url. @@ -3239,7 +3244,7 @@ Stopped_success_update_status_fail=Succeed in stopping th job, but failed to upd load_code_failed=Fail to load code, please check if the right branch is selected. error.dataset_select = dataset select error：the count exceed the limit or has same name -new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, and please put your model into %s then you can download it online +new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the environment %s, and please put your model into %s then you can download it online new_infer_gpu_tooltips = The dataset is stored in %s, the model file is stored in %s, please store the inference output in %s for subsequent downloads. [points] diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index f8bbb5ca9..2d75c2d05 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3100,6 +3100,11 @@ task_createmodel=`导入了新模型 %s` task_c2netnputrainjob=`创建了NPU类型训练任务 %s` task_c2netgputrainjob=`创建了CPU/GPU类型训练任务 %s` +binded_wechat=绑定微信 +dataset_recommended=`创建的数据集%s被设置为推荐数据集` +create_image=`提交了镜像%s` +image_recommended=`提交的镜像%s被设置为推荐镜像` +update_user_avatar=更新了头像 [tool] ago=%s前 @@ -3220,7 +3225,7 @@ card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 -train_dataset_path_rule = 数据集位置存储在环境变量data_url中，训练输出路径存储在环境变量train_url中。 +train_dataset_path_rule = 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 infer_dataset_path_rule = 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 view_sample = 查看样例 inference_output_path_rule = 推理输出路径存储在环境变量result_url中。 @@ -3258,7 +3263,7 @@ load_code_failed=代码加载失败，请确认选择了正确的分支。 error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 -new_train_gpu_tooltips =训练脚本存储在%s中，数据集存储在%s中，训练输出请存储在%s中以供后续下载。 +new_train_gpu_tooltips =训练脚本存储在%s中，数据集存储在%s中，预训练模型存放在环境变量%s中，训练输出请存储在%s中以供后续下载。 new_infer_gpu_tooltips = 数据集存储在%s中，模型文件存储在%s中，推理输出请存储在%s中以供后续下载。 [points] diff --git a/public/home/home.js b/public/home/home.js index 70b9d7253..c1849b3e3 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -163,6 +163,11 @@ document.onreadystatechange = function () { html += recordPrefix + actionName; html += " " + record.RefName + "" } + else if(record.OpType == "35"){ + var datasetLink = "" + record.Content.split('|')[1] + ""; + actionName = actionName.replace('{dataset}', datasetLink); + html += recordPrefix + actionName; + } else{ continue; } @@ -354,7 +359,10 @@ var actionNameZH={ "30":"导入了新模型", "31":"创建了CPU/GPU类型训练任务", "32":"创建了NPU类型训练任务", - "33":"创建了CPU/GPU类型训练任务" + "33":"创建了CPU/GPU类型训练任务", + "35":"创建的数据集 {dataset} 被设置为推荐数据集", + "36":"提交了镜像 {image}", + "37":"提交的镜像 {image} 被设置为推荐镜像", }; var actionNameEN={ @@ -382,7 +390,10 @@ var actionNameEN={ "30":" created new model", "31":" created CPU/GPU type training task", "32":" created NPU type training task", - "33":" created CPU/GPU type training task" + "33":" created CPU/GPU type training task", + "35":" created dataset {dataset} was set as recommended dataset", + "36":"committed image {image}", + "37":"committed image {image} was set as recommended image", }; var repoAndOrgZH={ diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 3c4a09b88..73aa9b791 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2769,7 +2769,7 @@ func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) { } } if form.CkptName != "" { - param += " --pretrainmodelname" + "=" + form.CkptName + param += " --ckpt_url" + "=" + "/pretrainmodel/" + form.CkptName } command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 08db4f512..088cf4557 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/services/reward/point/account" - "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/grampus" @@ -721,7 +720,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion req.PreTrainModelUrl = form.PreTrainModelUrl - + req.PreTrainModelPath = preTrainModelPath } err = grampus.GenerateTrainJob(ctx, req) @@ -986,8 +985,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" - commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) + commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" command += commandDownload } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" @@ -996,10 +994,14 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } //unzip code & dataset - unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) - - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand - command += commandUnzip + if processorType == grampus.ProcessorTypeNPU { + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" + command += commandUnzip + } else if processorType == grampus.ProcessorTypeGPU { + unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand + command += commandUnzip + } command += "echo \"unzip finished;start to exec code;\";" @@ -1029,14 +1031,13 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } } - if pretrainModelFileName != "" { - paramCode += " --pretrainmodelname" + "=" + pretrainModelFileName - } - var commandCode string if processorType == grampus.ProcessorTypeNPU { commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { + if pretrainModelFileName != "" { + paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName + } commandCode = "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" } @@ -1077,14 +1078,14 @@ func generateDatasetUnzipCommand(datasetName string) string { datasetNameArray := strings.Split(datasetName, ";") if len(datasetNameArray) == 1 { //单数据集 unZipDatasetCommand = "unzip -q '" + datasetName + "';" - if strings.HasSuffix(datasetName, ".tar.gz") { + if strings.HasSuffix(datasetNameArray[0], ".tar.gz") { unZipDatasetCommand = "tar --strip-components=1 -zxvf '" + datasetName + "';" } } else { //多数据集 for _, datasetNameTemp := range datasetNameArray { - if strings.HasSuffix(datasetName, ".tar.gz") { - unZipDatasetCommand = unZipDatasetCommand + "tar -zxvf '" + datasetName + "';" + if strings.HasSuffix(datasetNameTemp, ".tar.gz") { + unZipDatasetCommand = unZipDatasetCommand + "tar -zxvf '" + datasetNameTemp + "';" } else { unZipDatasetCommand = unZipDatasetCommand + "unzip -q '" + datasetNameTemp + "' -d './" + strings.TrimSuffix(datasetNameTemp, ".zip") + "';" } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 7e03f1633..465442d50 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1052,9 +1052,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.RenderWithErr("Resource specification not available", tplModelArtsTrainJobNew, &form) return } - if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice*form.WorkServerNumber) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + trainJobNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplModelArtsTrainJobNew, &form) return } diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index 88cd8b22a..c9ed207ad 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -141,7 +141,7 @@ func GetAdminRewardList(ctx *context.Context) { func buildAdminRewardRecordListOpts(ctx *context.Context) (*models.RewardRecordListOpts, error) { operateType := ctx.Query("operate") sourceType := ctx.Query("source") - actionType := ctx.QueryInt("action") + taskType := ctx.Query("action") serialNo := ctx.Query("serialNo") status := ctx.Query("status") @@ -161,7 +161,7 @@ func buildAdminRewardRecordListOpts(ctx *context.Context) (*models.RewardRecordL RewardType: models.RewardTypePoint, OrderBy: orderBy, SourceType: sourceType, - ActionType: actionType, + TaskType: taskType, SerialNo: serialNo, IsAdmin: true, Status: status, diff --git a/routers/task/config.go b/routers/task/config.go index 9740561d2..c8a994e39 100644 --- a/routers/task/config.go +++ b/routers/task/config.go @@ -13,11 +13,11 @@ import ( func GetTaskConfigList(ctx *context.Context) { page := ctx.QueryInt("Page") status := ctx.QueryInt("Status") - action := ctx.QueryInt("Action") + action := ctx.Query("Action") r, err := task.GetTaskConfigWithLimitList(models.GetTaskConfigOpts{ ListOptions: models.ListOptions{PageSize: 20, Page: page}, Status: status, - ActionType: action, + TaskType: action, }) if err != nil { log.Error("GetTaskConfigList error.%v", err) diff --git a/services/reward/cloudbrain_deduct.go b/services/reward/cloudbrain_deduct.go index f60c50aa1..510a96b32 100644 --- a/services/reward/cloudbrain_deduct.go +++ b/services/reward/cloudbrain_deduct.go @@ -25,20 +25,32 @@ func AcceptStatusChangeAction() { } func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.RewardPeriodicTask, error) { + sourceId := getCloudBrainPointTaskSourceId(task) + r, err := GetPeriodicTask(models.SourceTypeRunCloudbrainTask, sourceId, sourceId, models.OperateTypeDecrease) + if err != nil { + return nil, err + } + + if r != nil { + log.Debug("PeriodicTask is already exist.cloudbrain.ID = %d", task.ID) + return r, nil + } + if !setting.CloudBrainPaySwitch { + log.Debug("CloudBrainPaySwitch is off") return nil, nil } - unitPrice, err := models.GetCloudbrainTaskUnitPrice(task.ID) + unitPrice, err := models.GetCloudbrainTaskUnitPrice(task) if err != nil { return nil, err } if unitPrice == 0 { - log.Debug("finish StartAndGetCloudBrainPointDeductTask, UnitPrice = 0 task.ID=%d", task.ID) + log.Debug("Finish startAndGetCloudBrainPointDeductTask, UnitPrice = 0 task.ID=%d", task.ID) return nil, nil } - return StartAndGetPeriodicTask(&models.StartPeriodicTaskOpts{ + return StartPeriodicTask(&models.StartPeriodicTaskOpts{ SourceType: models.SourceTypeRunCloudbrainTask, SourceId: getCloudBrainPointTaskSourceId(task), TargetUserId: task.UserID, diff --git a/services/reward/operator.go b/services/reward/operator.go index b9d8c8d59..ec9a514a5 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -180,43 +180,41 @@ func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) er return nil } -func StartPeriodicTaskAsyn(opts *models.StartPeriodicTaskOpts) { - go StartAndGetPeriodicTask(opts) +func GetPeriodicTask(sourceType models.SourceType, sourceId, requestId string, operateType models.RewardOperateType) (*models.RewardPeriodicTask, error) { + _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType.Name(), requestId, operateType.Name()) + if err == nil { + task, err := models.GetPeriodicTaskBySourceIdAndType(sourceType, sourceId, operateType) + if err != nil { + log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) + return nil, err + } + return task, nil + } + + if err != nil && !models.IsErrRecordNotExist(err) { + log.Error("GetPointOperateRecordBySourceTypeAndRequestId error,%v", err) + return nil, err + } + return nil, nil } -func StartAndGetPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.RewardPeriodicTask, error) { - defer func() { - if err := recover(); err != nil { - combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) - log.Error("PANIC:%v", combinedErr) - } - }() +func StartPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.RewardPeriodicTask, error) { //add lock var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(opts.RequestId, opts.SourceType.Name(), opts.OperateType.Name())) isOk, err := rewardLock.Lock(3 * time.Second) - if err != nil { - log.Error("StartAndGetPeriodicTask RewardOperateLock error. %v", err) - return nil, err - } if !isOk { log.Info("duplicated operate request,targetUserId=%d requestId=%s", opts.TargetUserId, opts.RequestId) return nil, nil } defer rewardLock.UnLock() - _, err = models.GetPointOperateRecordBySourceTypeAndRequestId(opts.SourceType.Name(), opts.RequestId, opts.OperateType.Name()) - if err == nil { - task, err := models.GetPeriodicTaskBySourceIdAndType(opts.SourceType, opts.SourceId, opts.OperateType) - if err != nil { - log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) - return nil, err - } - return task, nil + r, err := GetPeriodicTask(opts.SourceType, opts.SourceId, opts.RequestId, opts.OperateType) + if err != nil { + return nil, err } - if err != nil && !models.IsErrRecordNotExist(err) { - log.Error("operate is handled error,%v", err) - return nil, err + if r != nil { + return r, nil } //new reward operate record diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 98bcc8a85..7470b1198 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35} type ClientsManager struct { Clients *orderedmap.OrderedMap @@ -107,16 +107,18 @@ func initActionQueue() { func filterUserPrivateInfo(action *models.Action) { action.Comment = nil - action.ActUser.Email = "" - action.ActUser.Passwd = "" - action.ActUser.PasswdHashAlgo = "" - action.ActUser.PrivateKey = "" - action.ActUser.PublicKey = "" - action.ActUser.Salt = "" - action.ActUser.FullName = "" - action.ActUser.AvatarEmail = "" - action.ActUser.IsAdmin = false - action.ActUser.EmailNotificationsPreference = "" - action.ActUser.IsOperator = false + if action.ActUser != nil { + action.ActUser.Email = "" + action.ActUser.Passwd = "" + action.ActUser.PasswdHashAlgo = "" + action.ActUser.PrivateKey = "" + action.ActUser.PublicKey = "" + action.ActUser.Salt = "" + action.ActUser.FullName = "" + action.ActUser.AvatarEmail = "" + action.ActUser.IsAdmin = false + action.ActUser.EmailNotificationsPreference = "" + action.ActUser.IsOperator = false + } } diff --git a/services/task/task.go b/services/task/task.go index 2ff2e13c4..194a43649 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -6,6 +6,8 @@ import ( "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/limiter" "fmt" + "strconv" + "strings" ) func Accomplish(action models.Action) { @@ -15,46 +17,78 @@ func Accomplish(action models.Action) { log.Error("PANIC:%v", combinedErr) } }() - action.OpType = models.GetTaskOptType(action) - switch action.OpType { + taskType := models.GetTaskTypeFromAction(action.OpType) + if taskType == "" { + log.Info("Accomplish finished.taskType is not exist.action.ID=%d", action.ID) + return + } + actions := make([]models.Action, 0) + actions = append(actions, action) + switch taskType { //only creating public repo can be rewarded - case models.ActionCreateRepo: + case models.TaskCreatePublicRepo: if action.Repo.IsPrivate { return } //only creating public image can be rewarded - case models.ActionCreateImage: + case models.TaskCreateImage: if action.IsPrivate { return } - case models.ActionBindWechat: + case models.TaskBindWechat: n, err := models.CountWechatBindLog(action.Content, models.WECHAT_BIND) if err != nil { log.Error("CountWechatBindLog error when accomplish task,err=%v", err) return } //if wechatOpenId has been bound before,the action can not get reward - if n > 1 { + if n > 1 && models.IsWechatOpenIdRewarded(action.Content) { + log.Debug("the wechat account has been bound before,wechatOpenId = %s", action.Content) return } + case models.TaskDatasetRecommended: + datasetIdStr := strings.Split(action.Content, "|")[0] + datasetId, _ := strconv.ParseInt(datasetIdStr, 10, 64) + users, err := models.GetAllDatasetContributorByDatasetId(datasetId) + if err != nil { + return + } + for _, user := range users { + if user.ID == action.ActUserID { + continue + } + actions = append(actions, models.Action{ + ID: action.ID, + OpType: models.ActionDatasetRecommended, + ActUserID: action.UserID, + UserID: user.ID, + RepoID: action.RepoID, + Content: action.Content, + }) + } + + } + batchAccomplish(taskType, actions...) +} +func batchAccomplish(taskType models.TaskType, actions ...models.Action) { + for _, act := range actions { + go accomplish(act, taskType) } - go accomplish(action) } -func accomplish(action models.Action) error { +func accomplish(action models.Action, taskType models.TaskType) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) log.Error("PANIC:%v", combinedErr) } }() - userId := action.ActUserID - taskType := fmt.Sprint(action.OpType) + userId := action.UserID //get task config - config, err := GetTaskConfig(taskType) + config, err := GetTaskConfig(string(taskType)) if err != nil { log.Error("GetTaskConfig error,%v", err) return err @@ -86,7 +120,7 @@ func accomplish(action models.Action) error { reward.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, SourceId: fmt.Sprint(action.ID), - SourceTemplateId: fmt.Sprint(action.OpType), + SourceTemplateId: string(taskType), Title: config.Title, Reward: models.Reward{ Amount: config.AwardAmount, diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index b07832752..607af5f07 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -122,7 +122,7 @@ {{template "custom/task_wait_count" .}}
- {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/code" "/dataset" "/model" | Safe}} + {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/code" "/dataset" "ckpt_url" "/model" | Safe}}
diff --git a/templates/repo/datasets/index.tmpl b/templates/repo/datasets/index.tmpl index eb7bcec21..b5ddff638 100755 --- a/templates/repo/datasets/index.tmpl +++ b/templates/repo/datasets/index.tmpl @@ -203,7 +203,6 @@
{{end}}
- {{if or (.dataset.Category) (.dataset.Task) (.dataset.License)}}
{{if .dataset.Category}} {{$category := .dataset.Category}} @@ -220,7 +219,6 @@ href="{{AppSubUrl}}/explore/datasets?sort={{$.SortType}}&q={{$.Keyword}}&tab={{$.TabName}}&category={{$.Category}}&task={{$.Task}}&license={{.dataset.License}}">{{.dataset.License}} {{end}}
- {{end}}
- {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/tmp/code" "/tmp/dataset" "/tmp/output" | Safe}} + {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/tmp/code" "/tmp/dataset" "ckpt_url" "/tmp/output" | Safe}}
diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index ff58e4f4b..92dca5bae 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -106,7 +106,7 @@ {{template "custom/task_wait_count" .}}
- {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/cache/code" "/cache/dataset" "/cache/output" | Safe}} + {{.i18n.Tr "cloudbrain.new_train_gpu_tooltips" "/cache/code" "/cache/dataset" "ckpt_url" "/cache/output" | Safe}}
diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index df1fb90a9..8e2810317 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -196,8 +196,8 @@
- {{if .bootFile}} - + {{if .boot_file}} + {{else}} {{end}} diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 125d11517..0c9bb3ba4 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -256,7 +256,7 @@ {{if .CanModify}} {{$.i18n.Tr "repo.modelarts.modify"}} + href="{{$.RepoLink}}/modelarts/train-job/{{.JobID}}/create_version?version_name={{.VersionName}}&path=show">{{$.i18n.Tr "repo.modelarts.modify"}} {{else}} {{$.i18n.Tr "repo.modelarts.modify"}} @@ -650,7 +650,7 @@
- +   diff --git a/templates/repo/modelarts/trainjob/version_new.tmpl b/templates/repo/modelarts/trainjob/version_new.tmpl index 856b307cf..89bf6fc08 100644 --- a/templates/repo/modelarts/trainjob/version_new.tmpl +++ b/templates/repo/modelarts/trainjob/version_new.tmpl @@ -261,7 +261,7 @@ - {{.i18n.Tr "repo.cloudbrain.cancel"}} + {{.i18n.Tr "repo.cloudbrain.cancel"}}
@@ -274,7 +274,11 @@