From 762b3b1c174786672bbcb511026b237e350a200e Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 19 Sep 2022 17:02:55 +0800 Subject: [PATCH 01/24] #1249 update point when multi node --- models/cloudbrain_spec.go | 10 +++++++--- routers/repo/modelarts.go | 2 +- services/reward/cloudbrain_deduct.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index a14e3f886..c32e4b0fd 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -123,10 +123,14 @@ func GetResourceSpecMapByCloudbrainIDs(ids []int64) (map[int64]*Specification, e return r, nil } -func GetCloudbrainTaskUnitPrice(cloudbrainId int64) (int, error) { - s, err := GetCloudbrainSpecByID(cloudbrainId) +func GetCloudbrainTaskUnitPrice(task Cloudbrain) (int, error) { + s, err := GetCloudbrainSpecByID(task.ID) if err != nil { return 0, err } - return s.UnitPrice, nil + var n = 1 + if task.WorkServerNumber > 1 { + n = task.WorkServerNumber + } + return s.UnitPrice * n, nil } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 4f3341e6f..19b316a3b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1216,7 +1216,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.RenderWithErr("Resource specification not available", tplModelArtsTrainJobNew, &form) return } - if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice*form.WorkServerNumber) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplModelArtsTrainJobNew, &form) diff --git a/services/reward/cloudbrain_deduct.go b/services/reward/cloudbrain_deduct.go index f60c50aa1..826b79bdc 100644 --- a/services/reward/cloudbrain_deduct.go +++ b/services/reward/cloudbrain_deduct.go @@ -29,7 +29,7 @@ func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.Rewar return nil, nil } - unitPrice, err := models.GetCloudbrainTaskUnitPrice(task.ID) + unitPrice, err := models.GetCloudbrainTaskUnitPrice(task) if err != nil { return nil, err } From 39f632b7dfecec3b31713a89dbb26779af814e4d Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Tue, 20 Sep 2022 10:49:14 +0800 Subject: [PATCH 02/24] #1249 update --- models/cloudbrain.go | 42 ++++++++++++++++++-------------- services/reward/cloudbrain_deduct.go | 16 +++++++++++-- services/reward/operator.go | 46 +++++++++++++++++------------------- 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e43b86030..6b51d11a8 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -201,16 +201,17 @@ type Cloudbrain struct { } type CloudbrainShow struct { - ID int64 - JobID string - RepoFullName string - Type int - JobType string - DisplayJobName string - Duration string - ResourceSpec *Specification - ComputeResource string - AiCenter string + ID int64 + JobID string + RepoFullName string + Type int + JobType string + DisplayJobName string + Duration string + ResourceSpec *Specification + ComputeResource string + AiCenter string + WorkServerNumber int } type CloudbrainShow4Action struct { @@ -223,15 +224,20 @@ type CloudbrainShow4Action struct { } func (task *Cloudbrain) ToShow() *CloudbrainShow { + n := 1 + if task.WorkServerNumber > 1 { + n = task.WorkServerNumber + } c := &CloudbrainShow{ - ID: task.ID, - JobID: task.JobID, - JobType: task.JobType, - Type: task.Type, - DisplayJobName: task.DisplayJobName, - Duration: task.TrainJobDuration, - ResourceSpec: task.Spec, - ComputeResource: task.ComputeResource, + ID: task.ID, + JobID: task.JobID, + JobType: task.JobType, + Type: task.Type, + DisplayJobName: task.DisplayJobName, + Duration: task.TrainJobDuration, + ResourceSpec: task.Spec, + ComputeResource: task.ComputeResource, + WorkServerNumber: n, } if task.Repo != nil { c.RepoFullName = task.Repo.FullName() diff --git a/services/reward/cloudbrain_deduct.go b/services/reward/cloudbrain_deduct.go index 826b79bdc..510a96b32 100644 --- a/services/reward/cloudbrain_deduct.go +++ b/services/reward/cloudbrain_deduct.go @@ -25,7 +25,19 @@ func AcceptStatusChangeAction() { } func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.RewardPeriodicTask, error) { + sourceId := getCloudBrainPointTaskSourceId(task) + r, err := GetPeriodicTask(models.SourceTypeRunCloudbrainTask, sourceId, sourceId, models.OperateTypeDecrease) + if err != nil { + return nil, err + } + + if r != nil { + log.Debug("PeriodicTask is already exist.cloudbrain.ID = %d", task.ID) + return r, nil + } + if !setting.CloudBrainPaySwitch { + log.Debug("CloudBrainPaySwitch is off") return nil, nil } @@ -34,11 +46,11 @@ func StartAndGetCloudBrainPointDeductTask(task models.Cloudbrain) (*models.Rewar return nil, err } if unitPrice == 0 { - log.Debug("finish StartAndGetCloudBrainPointDeductTask, UnitPrice = 0 task.ID=%d", task.ID) + log.Debug("Finish startAndGetCloudBrainPointDeductTask, UnitPrice = 0 task.ID=%d", task.ID) return nil, nil } - return StartAndGetPeriodicTask(&models.StartPeriodicTaskOpts{ + return StartPeriodicTask(&models.StartPeriodicTaskOpts{ SourceType: models.SourceTypeRunCloudbrainTask, SourceId: getCloudBrainPointTaskSourceId(task), TargetUserId: task.UserID, diff --git a/services/reward/operator.go b/services/reward/operator.go index b9d8c8d59..ec9a514a5 100644 --- a/services/reward/operator.go +++ b/services/reward/operator.go @@ -180,43 +180,41 @@ func UpdateRewardRecordToFinalStatus(sourceType, requestId, newStatus string) er return nil } -func StartPeriodicTaskAsyn(opts *models.StartPeriodicTaskOpts) { - go StartAndGetPeriodicTask(opts) +func GetPeriodicTask(sourceType models.SourceType, sourceId, requestId string, operateType models.RewardOperateType) (*models.RewardPeriodicTask, error) { + _, err := models.GetPointOperateRecordBySourceTypeAndRequestId(sourceType.Name(), requestId, operateType.Name()) + if err == nil { + task, err := models.GetPeriodicTaskBySourceIdAndType(sourceType, sourceId, operateType) + if err != nil { + log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) + return nil, err + } + return task, nil + } + + if err != nil && !models.IsErrRecordNotExist(err) { + log.Error("GetPointOperateRecordBySourceTypeAndRequestId error,%v", err) + return nil, err + } + return nil, nil } -func StartAndGetPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.RewardPeriodicTask, error) { - defer func() { - if err := recover(); err != nil { - combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) - log.Error("PANIC:%v", combinedErr) - } - }() +func StartPeriodicTask(opts *models.StartPeriodicTaskOpts) (*models.RewardPeriodicTask, error) { //add lock var rewardLock = redis_lock.NewDistributeLock(redis_key.RewardOperateLock(opts.RequestId, opts.SourceType.Name(), opts.OperateType.Name())) isOk, err := rewardLock.Lock(3 * time.Second) - if err != nil { - log.Error("StartAndGetPeriodicTask RewardOperateLock error. %v", err) - return nil, err - } if !isOk { log.Info("duplicated operate request,targetUserId=%d requestId=%s", opts.TargetUserId, opts.RequestId) return nil, nil } defer rewardLock.UnLock() - _, err = models.GetPointOperateRecordBySourceTypeAndRequestId(opts.SourceType.Name(), opts.RequestId, opts.OperateType.Name()) - if err == nil { - task, err := models.GetPeriodicTaskBySourceIdAndType(opts.SourceType, opts.SourceId, opts.OperateType) - if err != nil { - log.Error("GetPeriodicTaskBySourceIdAndType error,%v", err) - return nil, err - } - return task, nil + r, err := GetPeriodicTask(opts.SourceType, opts.SourceId, opts.RequestId, opts.OperateType) + if err != nil { + return nil, err } - if err != nil && !models.IsErrRecordNotExist(err) { - log.Error("operate is handled error,%v", err) - return nil, err + if r != nil { + return r, nil } //new reward operate record From fbde5034d7173931d22766b2ec09130fa3edae98 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Tue, 20 Sep 2022 11:20:58 +0800 Subject: [PATCH 03/24] update --- web_src/vuepages/pages/reward/point/utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web_src/vuepages/pages/reward/point/utils.js b/web_src/vuepages/pages/reward/point/utils.js index ae6a8f181..039804066 100644 --- a/web_src/vuepages/pages/reward/point/utils.js +++ b/web_src/vuepages/pages/reward/point/utils.js @@ -60,7 +60,9 @@ const renderSpecStr = (spec, showPoint) => { var ngpu = `${spec.ComputeResource}: ${spec.AccCardsNum + '*' + getListValueWithKey(ACC_CARD_TYPE, spec.AccCardType)}`; var gpuMemStr = spec.GPUMemGiB != 0 ? `${i18n.t('resourcesManagement.gpuMem')}: ${spec.GPUMemGiB}GB, ` : ''; var sharedMemStr = spec.ShareMemGiB != 0 ? `, ${i18n.t('resourcesManagement.shareMem')}: ${spec.ShareMemGiB}GB` : ''; - var pointStr = showPoint ? `, ${spec.UnitPrice == 0 ? i18n.t('resourcesManagement.free') : spec.UnitPrice + i18n.t('resourcesManagement.point_hr')}` : ''; + var workServerNum = spec.workServerNumber; + var workServerNumStr = showPoint && workServerNum != 1 && spec.UnitPrice != 0 ? workServerNum + '*' : ''; + var pointStr = showPoint ? `, ${spec.UnitPrice == 0 ? i18n.t('resourcesManagement.free') : workServerNumStr + spec.UnitPrice + i18n.t('resourcesManagement.point_hr')}` : ''; var specStr = `${ngpu}, CPU: ${spec.CpuCores}, ${gpuMemStr}${i18n.t('resourcesManagement.mem')}: ${spec.MemGiB}GB${sharedMemStr}${pointStr}`; return specStr; }; @@ -154,6 +156,7 @@ export const getRewardPointRecordInfo = (record) => { // } const resourceSpec = record?.Cloudbrain?.ResourceSpec; if (resourceSpec) { + resourceSpec.workServerNumber = record?.Cloudbrain?.workServerNumber || 1; out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${renderSpecStr(resourceSpec, true)}】`; } } From 1bfc7532ce92bef3c466333fc18a2c8f51b9b68f Mon Sep 17 00:00:00 2001 From: chenshihai Date: Tue, 20 Sep 2022 11:40:02 +0800 Subject: [PATCH 04/24] update --- web_src/vuepages/langs/config/en-US.js | 1 + web_src/vuepages/langs/config/zh-CN.js | 1 + web_src/vuepages/pages/reward/point/utils.js | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js index aa5a77909..f306adb2a 100644 --- a/web_src/vuepages/langs/config/en-US.js +++ b/web_src/vuepages/langs/config/en-US.js @@ -129,6 +129,7 @@ const en = { shareMem: 'Share Memory', unitPrice: 'Unit Price', point_hr: 'Point/hr', + node: 'node', free: 'Free', onShelfConfirm: 'Are you sure to on shelf the resources specification?', offShelfConfirm: 'Are you sure to off shelf the resources specification?', diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js index 4fc537a33..83a1a9343 100644 --- a/web_src/vuepages/langs/config/zh-CN.js +++ b/web_src/vuepages/langs/config/zh-CN.js @@ -129,6 +129,7 @@ const zh = { shareMem: '共享内存', unitPrice: '单价', point_hr: '积分/时', + node: '节点', free: '免费', onShelfConfirm: '请确认上架该规格？', offShelfConfirm: '请确认下架该规格？', diff --git a/web_src/vuepages/pages/reward/point/utils.js b/web_src/vuepages/pages/reward/point/utils.js index 039804066..c26862aa6 100644 --- a/web_src/vuepages/pages/reward/point/utils.js +++ b/web_src/vuepages/pages/reward/point/utils.js @@ -61,8 +61,8 @@ const renderSpecStr = (spec, showPoint) => { var gpuMemStr = spec.GPUMemGiB != 0 ? `${i18n.t('resourcesManagement.gpuMem')}: ${spec.GPUMemGiB}GB, ` : ''; var sharedMemStr = spec.ShareMemGiB != 0 ? `, ${i18n.t('resourcesManagement.shareMem')}: ${spec.ShareMemGiB}GB` : ''; var workServerNum = spec.workServerNumber; - var workServerNumStr = showPoint && workServerNum != 1 && spec.UnitPrice != 0 ? workServerNum + '*' : ''; - var pointStr = showPoint ? `, ${spec.UnitPrice == 0 ? i18n.t('resourcesManagement.free') : workServerNumStr + spec.UnitPrice + i18n.t('resourcesManagement.point_hr')}` : ''; + var workServerNumStr = showPoint && workServerNum != 1 && spec.UnitPrice != 0 ? '*' + workServerNum + i18n.t('resourcesManagement.node') : ''; + var pointStr = showPoint ? `, ${spec.UnitPrice == 0 ? i18n.t('resourcesManagement.free') : spec.UnitPrice + i18n.t('resourcesManagement.point_hr') + workServerNumStr}` : ''; var specStr = `${ngpu}, CPU: ${spec.CpuCores}, ${gpuMemStr}${i18n.t('resourcesManagement.mem')}: ${spec.MemGiB}GB${sharedMemStr}${pointStr}`; return specStr; }; @@ -156,7 +156,7 @@ export const getRewardPointRecordInfo = (record) => { // } const resourceSpec = record?.Cloudbrain?.ResourceSpec; if (resourceSpec) { - resourceSpec.workServerNumber = record?.Cloudbrain?.workServerNumber || 1; + resourceSpec.workServerNumber = record?.Cloudbrain?.WorkServerNumber || 1; out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${renderSpecStr(resourceSpec, true)}】`; } } From 44351bffded93a6522396881d6414862a55a5f1b Mon Sep 17 00:00:00 2001 From: chenshihai Date: Tue, 20 Sep 2022 15:24:07 +0800 Subject: [PATCH 05/24] update --- templates/reward/point/rule.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/reward/point/rule.tmpl b/templates/reward/point/rule.tmpl index 3d81fd0f7..adc074faa 100644 --- a/templates/reward/point/rule.tmpl +++ b/templates/reward/point/rule.tmpl @@ -133,7 +133,7 @@ if (taskRule.TaskCode == key) { ruleTr.find('.point').text(taskRule.AwardAmount); ruleTr.find('.typ').text(taskRule.RefreshRate == 'DAILY' ? '每日' : taskRule.RefreshRate == 'NOT_CYCLE' ? '累计' : taskRule.RefreshRate); - ruleTr.find('.limit').text(taskRule.LimitNum); + ruleTr.find('.limit').text(taskRule.AwardAmount * taskRule.LimitNum); findOr = true; break; } From e4890a9bfa2af590288e4884825f61becca5f2d6 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 21 Sep 2022 10:45:08 +0800 Subject: [PATCH 06/24] #1249 update --- models/action.go | 23 ++++-------- models/reward_operate_record.go | 7 ++-- models/task_config.go | 69 ++++++++++++++++++++++++++++++++--- modules/notification/action/action.go | 16 -------- routers/reward/point/point.go | 4 +- routers/task/config.go | 4 +- services/task/task.go | 24 +++++++----- 7 files changed, 92 insertions(+), 55 deletions(-) diff --git a/models/action.go b/models/action.go index db3114c95..d0a763e08 100755 --- a/models/action.go +++ b/models/action.go @@ -61,13 +61,10 @@ const ( ActionCreateGrampusNPUTrainTask //32 ActionCreateGrampusGPUTrainTask //33 ActionBindWechat //34 - ActionCreateCloudbrainTask //35 - ActionDatasetRecommended //36 - ActionCreateImage //37 - ActionImageRecommend //38 - ActionChangeUserAvatar //39 - ActionPushCommits //40 - ActionForkRepo //41 + ActionDatasetRecommended //35 + ActionCreateImage //36 + ActionImageRecommend //37 + ActionChangeUserAvatar //38 ) @@ -95,6 +92,7 @@ type Action struct { type ActionShow struct { OpType ActionType + TaskType TaskType RepoLink string ShortRepoFullDisplayName string Content string @@ -241,7 +239,8 @@ func (a *Action) GetRepoLink() string { func (a *Action) ToShow() *ActionShow { actionShow := &ActionShow{} - actionShow.OpType = GetTaskOptType(*a) + actionShow.OpType = a.OpType + actionShow.TaskType = GetTaskTypeFromAction(a.OpType) actionShow.Content = a.Content actionShow.RefName = a.RefName @@ -272,14 +271,6 @@ func (a *Action) ToShow() *ActionShow { return actionShow } -func GetTaskOptType(action Action) ActionType { - //Convert all types of cloudbrain tasks action into ActionCreateCloudbrainTask - if action.IsCloudbrainAction() { - return ActionCreateCloudbrainTask - } - return action.OpType -} - // GetRepositoryFromMatch returns a *Repository from a username and repo strings func GetRepositoryFromMatch(ownerName string, repoName string) (*Repository, error) { var err error diff --git a/models/reward_operate_record.go b/models/reward_operate_record.go index ad00cc05c..1992eccba 100644 --- a/models/reward_operate_record.go +++ b/models/reward_operate_record.go @@ -2,7 +2,6 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "fmt" "strconv" "strings" "xorm.io/builder" @@ -355,7 +354,7 @@ type RewardRecordListOpts struct { OperateType RewardOperateType RewardType RewardType SourceType string - ActionType int + TaskType string SerialNo string OrderBy RewardOperateOrderBy IsAdmin bool @@ -381,8 +380,8 @@ func (opts *RewardRecordListOpts) toCond() builder.Cond { if opts.SourceType != "" { cond = cond.And(builder.Eq{"reward_operate_record.source_type": opts.SourceType}) } - if opts.ActionType > 0 { - cond = cond.And(builder.Eq{"reward_operate_record.source_template_id": fmt.Sprint(opts.ActionType)}) + if opts.TaskType != "" { + cond = cond.And(builder.Eq{"reward_operate_record.source_template_id": opts.TaskType}) } if opts.SerialNo != "" { cond = cond.And(builder.Like{"reward_operate_record.serial_no", opts.SerialNo}) diff --git a/models/task_config.go b/models/task_config.go index baa8c9adb..8650851be 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -2,7 +2,6 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" - "fmt" "xorm.io/builder" ) @@ -11,6 +10,66 @@ const ( PeriodDaily = "DAILY" ) +type TaskType string + +const ( + TaskCreatePublicRepo TaskType = "CreatePublicRepo" + TaskCreateIssue TaskType = "CreateIssue" + TaskCreatePullRequest TaskType = "CreatePullRequest" + TaskCommentIssue TaskType = "CommentIssue" + TaskUploadAttachment TaskType = "UploadAttachment" + TaskCreateNewModelTask TaskType = "CreateNewModelTask" + TaskBindWechat TaskType = "BindWechat" + TaskCreateCloudbrainTask TaskType = "CreateCloudbrainTask" + TaskDatasetRecommended TaskType = "DatasetRecommended " + TaskCreateImage TaskType = "CreateImage" + TaskImageRecommend TaskType = "ImageRecommend" + TaskChangeUserAvatar TaskType = "ChangeUserAvatar" + TaskPushCommits TaskType = "PushCommits" +) + +func GetTaskTypeFromAction(a ActionType) TaskType { + switch a { + case ActionCreateDebugGPUTask, + ActionCreateDebugNPUTask, + ActionCreateTrainTask, + ActionCreateInferenceTask, + ActionCreateBenchMarkTask, + ActionCreateGPUTrainTask, + ActionCreateGrampusNPUTrainTask, + ActionCreateGrampusGPUTrainTask: + return TaskCreateCloudbrainTask + case ActionCreateRepo: + return TaskCreatePublicRepo + case ActionCreatePullRequest: + return TaskCreatePullRequest + case ActionCommentIssue: + return TaskCommentIssue + case ActionUploadAttachment: + return TaskUploadAttachment + case ActionCreateNewModelTask: + return TaskCreateNewModelTask + case ActionBindWechat: + return TaskBindWechat + case ActionDatasetRecommended: + return TaskDatasetRecommended + case ActionImageRecommend: + return TaskImageRecommend + case ActionCreateImage: + return TaskCreateImage + case ActionChangeUserAvatar: + return TaskChangeUserAvatar + case ActionCommitRepo, + ActionDeleteBranch, + ActionPushTag, + ActionDeleteTag: + return TaskPushCommits + case ActionCreateIssue: + return TaskCreateIssue + } + return "" +} + //PointTaskConfig Only add and delete are allowed, edit is not allowed //so if you want to edit config for some task code,please delete first and add new one type TaskConfig struct { @@ -124,8 +183,8 @@ func GetTaskConfigList() ([]*TaskConfig, error) { type GetTaskConfigOpts struct { ListOptions - Status int //1 normal 2 deleted - ActionType int + Status int //1 normal 2 deleted + TaskType string } func GetTaskConfigPageWithDeleted(opt GetTaskConfigOpts) ([]*TaskAndLimiterConfig, int64, error) { @@ -133,8 +192,8 @@ func GetTaskConfigPageWithDeleted(opt GetTaskConfigOpts) ([]*TaskAndLimiterConfi opt.Page = 1 } cond := builder.NewCond() - if opt.ActionType > 0 { - cond = cond.And(builder.Eq{"task_code": fmt.Sprint(opt.ActionType)}) + if opt.TaskType != "" { + cond = cond.And(builder.Eq{"task_code": opt.TaskType}) } var count int64 diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index bfe574328..84181c555 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -428,19 +428,3 @@ func (t *actionNotifier) NotifyChangeUserAvatar(user *models.User, form auth.Ava log.Error("notifyWatchers: %v", err) } } - -func (t *actionNotifier) NotifyPushCommits(pusher *models.User, repo *models.Repository, refName, oldCommitID, newCommitID string, commits *repository.PushCommits) { - act := &models.Action{ - ActUserID: pusher.ID, - ActUser: pusher, - OpType: models.ActionPushCommits, - RepoID: repo.ID, - Repo: repo, - RefName: refName, - IsPrivate: repo.IsPrivate, - Content: fmt.Sprintf("%s|%s", oldCommitID, newCommitID), - } - if err := models.NotifyWatchers(act); err != nil { - log.Error("notifyWatchers: %v", err) - } -} diff --git a/routers/reward/point/point.go b/routers/reward/point/point.go index 88cd8b22a..c9ed207ad 100644 --- a/routers/reward/point/point.go +++ b/routers/reward/point/point.go @@ -141,7 +141,7 @@ func GetAdminRewardList(ctx *context.Context) { func buildAdminRewardRecordListOpts(ctx *context.Context) (*models.RewardRecordListOpts, error) { operateType := ctx.Query("operate") sourceType := ctx.Query("source") - actionType := ctx.QueryInt("action") + taskType := ctx.Query("action") serialNo := ctx.Query("serialNo") status := ctx.Query("status") @@ -161,7 +161,7 @@ func buildAdminRewardRecordListOpts(ctx *context.Context) (*models.RewardRecordL RewardType: models.RewardTypePoint, OrderBy: orderBy, SourceType: sourceType, - ActionType: actionType, + TaskType: taskType, SerialNo: serialNo, IsAdmin: true, Status: status, diff --git a/routers/task/config.go b/routers/task/config.go index 9740561d2..c8a994e39 100644 --- a/routers/task/config.go +++ b/routers/task/config.go @@ -13,11 +13,11 @@ import ( func GetTaskConfigList(ctx *context.Context) { page := ctx.QueryInt("Page") status := ctx.QueryInt("Status") - action := ctx.QueryInt("Action") + action := ctx.Query("Action") r, err := task.GetTaskConfigWithLimitList(models.GetTaskConfigOpts{ ListOptions: models.ListOptions{PageSize: 20, Page: page}, Status: status, - ActionType: action, + TaskType: action, }) if err != nil { log.Error("GetTaskConfigList error.%v", err) diff --git a/services/task/task.go b/services/task/task.go index 2ff2e13c4..5a2028572 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -15,19 +15,24 @@ func Accomplish(action models.Action) { log.Error("PANIC:%v", combinedErr) } }() - action.OpType = models.GetTaskOptType(action) - switch action.OpType { + taskType := models.GetTaskTypeFromAction(action.OpType) + if taskType == "" { + log.Info("Accomplish finished.taskType is not exist.action.ID=%d", action.ID) + return + } + + switch taskType { //only creating public repo can be rewarded - case models.ActionCreateRepo: + case models.TaskCreatePublicRepo: if action.Repo.IsPrivate { return } //only creating public image can be rewarded - case models.ActionCreateImage: + case models.TaskCreateImage: if action.IsPrivate { return } - case models.ActionBindWechat: + case models.TaskBindWechat: n, err := models.CountWechatBindLog(action.Content, models.WECHAT_BIND) if err != nil { log.Error("CountWechatBindLog error when accomplish task,err=%v", err) @@ -40,10 +45,10 @@ func Accomplish(action models.Action) { } } - go accomplish(action) + go accomplish(action, taskType) } -func accomplish(action models.Action) error { +func accomplish(action models.Action, taskType models.TaskType) error { defer func() { if err := recover(); err != nil { combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) @@ -51,10 +56,9 @@ func accomplish(action models.Action) error { } }() userId := action.ActUserID - taskType := fmt.Sprint(action.OpType) //get task config - config, err := GetTaskConfig(taskType) + config, err := GetTaskConfig(string(taskType)) if err != nil { log.Error("GetTaskConfig error,%v", err) return err @@ -86,7 +90,7 @@ func accomplish(action models.Action) error { reward.Operate(&models.RewardOperateContext{ SourceType: models.SourceTypeAccomplishTask, SourceId: fmt.Sprint(action.ID), - SourceTemplateId: fmt.Sprint(action.OpType), + SourceTemplateId: string(taskType), Title: config.Title, Reward: models.Reward{ Amount: config.AwardAmount, From 4b2f8e74dd17b2837648b1f69074b34f3ece0f3d Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 21 Sep 2022 17:00:03 +0800 Subject: [PATCH 07/24] update activity --- options/locale/locale_en-US.ini | 5 ++ options/locale/locale_zh-CN.ini | 5 ++ public/home/home.js | 25 +++++++++- templates/reward/point/rule.tmpl | 26 +++++----- templates/user/dashboard/feeds.tmpl | 10 ++++ web_src/vuepages/const/index.js | 4 +- web_src/vuepages/langs/config/en-US.js | 3 ++ web_src/vuepages/langs/config/zh-CN.js | 3 ++ web_src/vuepages/pages/reward/point/utils.js | 71 +++++++++++++++++----------- 9 files changed, 108 insertions(+), 44 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8d3868b73..8746f3590 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3074,6 +3074,11 @@ task_createmodel=`created new model %s` task_c2netnputrainjob=`created NPU training task %s` task_c2netgputrainjob=`created CPU/GPU training task %s` +binded_wechat=binded WeChat +dataset_recommended=`created dataset %s was set as recommended dataset` +create_image=`committed image %s` +image_recommended=`committed image %s was set as recommended image` +update_user_avatar=updated avatar [tool] ago = %s ago diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index b02600803..95710d09d 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3091,6 +3091,11 @@ task_createmodel=`导入了新模型 %s` task_c2netnputrainjob=`创建了NPU类型训练任务 %s` task_c2netgputrainjob=`创建了CPU/GPU类型训练任务 %s` +binded_wechat=绑定微信 +dataset_recommended=`创建的数据集%s被设置为推荐数据集` +create_image=`提交了镜像%s` +image_recommended=`提交的镜像%s被设置为推荐镜像` +update_user_avatar=更新了头像 [tool] ago=%s前 diff --git a/public/home/home.js b/public/home/home.js index 70b9d7253..4127362c0 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -163,6 +163,21 @@ document.onreadystatechange = function () { html += recordPrefix + actionName; html += " " + record.RefName + "" } + else if(record.OpType == "35"){ + var datasetLink = ''; + actionName.replace('{dataset}', datasetLink); + html += recordPrefix + actionName; + } + else if(record.OpType == "36"){ + var imageLink = ''; + actionName.replace('{image}', imageLink); + html += recordPrefix + actionName; + } + else if(record.OpType == "37"){ + var imageLink = ''; + actionName.replace('{image}', imageLink); + html += recordPrefix + actionName; + } else{ continue; } @@ -354,7 +369,10 @@ var actionNameZH={ "30":"导入了新模型", "31":"创建了CPU/GPU类型训练任务", "32":"创建了NPU类型训练任务", - "33":"创建了CPU/GPU类型训练任务" + "33":"创建了CPU/GPU类型训练任务", + "35":"创建的数据集 {dataset} 被设置为推荐数据集", + "36":"提交了镜像 {image}", + "37":"提交的镜像 {image} 被设置为推荐镜像", }; var actionNameEN={ @@ -382,7 +400,10 @@ var actionNameEN={ "30":" created new model", "31":" created CPU/GPU type training task", "32":" created NPU type training task", - "33":" created CPU/GPU type training task" + "33":" created CPU/GPU type training task", + "35":" created dataset {dataset} was set as recommended dataset", + "36":"committed image {image}", + "37":"committed image {image} was set as recommended image", }; var repoAndOrgZH={ diff --git a/templates/reward/point/rule.tmpl b/templates/reward/point/rule.tmpl index adc074faa..6e7a0a8cc 100644 --- a/templates/reward/point/rule.tmpl +++ b/templates/reward/point/rule.tmpl @@ -27,79 +27,79 @@ 上限值 奖励细节澄清 - + 完成微信扫码验证 - 累计积分获取上限 - 1、首次完成微信扫码验证，即获取积分。
2、同个账号，更换微信号码再验证不重复给积分。
3、同一个微信，绑定第一个账号时奖励积分，下次绑定其他账号时不再奖励。 - + 创建或Fork公开项目 - 每日积分获取上限 - 请注意项目质量，请勿复制粘贴或者重复公开项目，任何非常规的以公开项目去获取积分的行为将被认定为积分舞弊，将扣除所有积分。 - + 每日提出PR - 每日积分获取上限 - - + 每日commit - 每日积分获取上限 - 通过前台界面和后台命令行方式commit，都可获得奖励积分。 - + 每日提出任务 - 每日积分获取上限 - - + 发表评论 - 每日积分获取上限 - 禁止空评论或评论后马上删除等非正常获取积分的方式，一经发现将扣除所有积分。 - + 上传数据集文件 - 每日积分获取上限 - 请注意数据集质量，请勿复制粘贴或者重复公开数据集，任何非常规的以公开数据集去获取积分的行为将被认定为积分舞弊，将扣除所有积分。 - + 数据集被平台推荐 - 每日积分获取上限 - 仅统计属于个人的数据集，属于组织的数据集暂不统计。 - + 导入新模型 - 每日积分获取上限 - 请注意模型质量，请勿重复导入相同模型，任何非常规的以导入新模型去获取 积分的行为将被认定为积分舞弊，将扣除所有积分。 - + 每日运行云脑任务 - 每日积分获取上限 - 每日运行调试、训练、推理、评测中任何一种任务，即可获得。 - + 提交新公开镜像 - 每日积分获取上限 - - + 镜像被平台推荐 - 每日积分获取上限 - - + 首次更换头像 - 累计积分获取上限 - diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index e43191470..f3ec10c4a 100755 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -90,6 +90,16 @@ {{$.i18n.Tr "action.task_c2netnputrainjob" .GetRepoLink .Content .RefName | Str2html}} {{else if eq .GetOpType 33}} {{$.i18n.Tr "action.task_c2netgputrainjob" .GetRepoLink .Content .RefName | Str2html}} + {{else if eq .GetOpType 34}} + {{$.i18n.Tr "action.binded_wechat"}} + {{else if eq .GetOpType 35}} + {{$.i18n.Tr "action.dataset_recommended" .GetRepoLink | Str2html}} + {{else if eq .GetOpType 36}} + {{$.i18n.Tr "action.create_image" .GetRepoLink | Str2html}} + {{else if eq .GetOpType 37}} + {{$.i18n.Tr "action.image_recommended" .GetRepoLink | Str2html}} + {{else if eq .GetOpType 38}} + {{$.i18n.Tr "action.update_user_avatar"}} {{end}}

{{if or (eq .GetOpType 5) (eq .GetOpType 18)}} diff --git a/web_src/vuepages/const/index.js b/web_src/vuepages/const/index.js index fe133fb87..4051dbb17 100644 --- a/web_src/vuepages/const/index.js +++ b/web_src/vuepages/const/index.js @@ -3,8 +3,8 @@ import { i18n } from '~/langs'; export const SOURCE_TYPE = [{ k: 'ACCOMPLISH_TASK', v: i18n.t('accomplishTask') }, { k: 'ADMIN_OPERATE', v: i18n.t('adminOperate') }, { k: 'RUN_CLOUDBRAIN_TASK', v: i18n.t('runCloudBrainTask') }]; export const CONSUME_STATUS = [{ k: 'OPERATING', v: i18n.t('operating') }, { k: 'SUCCEEDED', v: i18n.t('succeeded') }]; export const POINT_ACTIONS = [ - { k: 1, v: i18n.t('createPublicProject') }, { k: 6, v: i18n.t('dailyPutforwardTasks') }, { k: 7, v: i18n.t('dailyPR') }, { k: 10, v: i18n.t('comment') }, { k: 24, v: i18n.t('uploadDatasetFile') }, { k: 30, v: i18n.t('importNewModel') }, { k: 34, v: i18n.t('completeWechatCodeScanningVerification') }, - { k: 35, v: i18n.t('dailyRunCloudbrainTasks') }, { k: 36, v: i18n.t('datasetRecommendedByThePlatform') }, { k: 37, v: i18n.t('submitNewPublicImage') }, { k: 38, v: i18n.t('imageRecommendedByThePlatform') }, { k: 39, v: i18n.t('firstChangeofAvatar') }, { k: 40, v: i18n.t('dailyCommit') }, + { k: 'CreatePublicRepo', v: i18n.t('createPublicProject') }, { k: 'CreateIssue', v: i18n.t('dailyPutforwardTasks') }, { k: 'CreatePullRequest', v: i18n.t('dailyPR') }, { k: 'CommentIssue', v: i18n.t('comment') }, { k: 'UploadAttachment', v: i18n.t('uploadDatasetFile') }, { k: 'CreateNewModelTask', v: i18n.t('importNewModel') }, { k: 'BindWechat', v: i18n.t('completeWechatCodeScanningVerification') }, + { k: 'CreateCloudbrainTask', v: i18n.t('dailyRunCloudbrainTasks') }, { k: 'DatasetRecommended', v: i18n.t('datasetRecommendedByThePlatform') }, { k: 'CreateImage', v: i18n.t('submitNewPublicImage') }, { k: 'ImageRecommend', v: i18n.t('imageRecommendedByThePlatform') }, { k: 'ChangeUserAvatar', v: i18n.t('firstChangeofAvatar') }, { k: 'PushCommits', v: i18n.t('dailyCommit') }, ]; export const JOB_TYPE = [{ k: 'DEBUG', v: i18n.t('debugTask') }, { k: 'TRAIN', v: i18n.t('trainTask') }, { k: 'INFERENCE', v: i18n.t('inferenceTask') }, { k: 'BENCHMARK', v: i18n.t('benchmarkTask') }]; diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js index f306adb2a..dafb374d4 100644 --- a/web_src/vuepages/langs/config/en-US.js +++ b/web_src/vuepages/langs/config/en-US.js @@ -69,6 +69,9 @@ const en = { setAsRecommendedImage: ' was set as recommended image', updatedAvatar: 'updated avatar', pushedBranch: 'pushed to {branch} at ', + deleteBranch: 'deleted branch {branch} from {repo}', + pushedTag: ' pushed tag {tag} to ', + deleteTag: ' deleted tag {tag} from {repo}', dailyMaxTips: `can't get full points when reach the daily upper limit`, memory: 'Memory', sharedMemory: 'Shared Memory', diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js index 83a1a9343..f190a77b3 100644 --- a/web_src/vuepages/langs/config/zh-CN.js +++ b/web_src/vuepages/langs/config/zh-CN.js @@ -69,6 +69,9 @@ const zh = { setAsRecommendedImage: '被设置为推荐镜像', updatedAvatar: '更新了头像', pushedBranch: '推送了{branch}分支代码到', + deleteBranch: '从{repo}删除分支{branch}', + pushedTag: '推送了标签{tag}到', + deleteTag: '从{repo}删除了标签{tag}', dailyMaxTips: '达到每日上限积分，不能拿满分', memory: '内存', sharedMemory: '共享内存', diff --git a/web_src/vuepages/pages/reward/point/utils.js b/web_src/vuepages/pages/reward/point/utils.js index c26862aa6..96e78a137 100644 --- a/web_src/vuepages/pages/reward/point/utils.js +++ b/web_src/vuepages/pages/reward/point/utils.js @@ -79,7 +79,7 @@ export const getRewardPointRecordInfo = (record) => { duration: record?.Cloudbrain?.Duration || '--', taskName: record?.Cloudbrain?.DisplayJobName || '--', taskId: record?.Cloudbrain?.ID, - action: record?.Action?.OpType ? getPointAction(record.Action.OpType) : '--', + action: record?.Action?.TaskType ? getPointAction(record.Action.TaskType) : '--', remark: record.Remark, amount: record.Amount, }; @@ -87,49 +87,72 @@ export const getRewardPointRecordInfo = (record) => { if (record.SourceType === 'ADMIN_OPERATE') { out.remark = record.Remark; } else if (record.SourceType === 'ACCOMPLISH_TASK') { - switch (record?.Action?.OpType) { - case 1: // 创建公开项目 - 创建了项目OpenI/aiforge + switch (record?.Action?.TaskType) { + case 'CreatePublicRepo': // 创建公开项目 - 创建了项目OpenI/aiforge out.remark = `${i18n.t('createdRepository')}${record.Action.ShortRepoFullDisplayName}`; break; - case 6: // 每日提出任务 - 创建了任务PCL-Platform.Intelligence/AISynergy#19 + case 'CreateIssue': // 每日提出任务 - 创建了任务PCL-Platform.Intelligence/AISynergy#19 out.remark = `${i18n.t('openedIssue')}${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; break; - case 7: // 每日提出PR - 创建了合并请求OpenI/aiforge#1 + case 'CreatePullRequest': // 每日提出PR - 创建了合并请求OpenI/aiforge#1 out.remark = `${i18n.t('createdPullRequest')}${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; break; - case 10: // 发表评论 - 评论了任务PCL-Platform.Intelligence/AISynergy#19 + case 'CommentIssue': // 发表评论 - 评论了任务PCL-Platform.Intelligence/AISynergy#19 out.remark = `${i18n.t('commentedOnIssue')}${record.Action.ShortRepoFullDisplayName}#${record.Action.IssueInfos[0]}`; break; - case 24: // 上传数据集文件 - 上传了数据集文件MMISTData.zip + case 'UploadAttachment': // 上传数据集文件 - 上传了数据集文件MMISTData.zip out.remark = `${i18n.t('uploadDataset')}${record.Action.RefName}`; break; - case 30: // 导入新模型 - 导入了新模型resnet50_qx7l + case 'CreateNewModelTask': // 导入新模型 - 导入了新模型resnet50_qx7l out.remark = `${i18n.t('createdNewModel')}${record.Action.RefName}`; break; - case 34: // 完成微信扫码验证 - 首次绑定微信奖励 + case 'BindWechat': // 完成微信扫码验证 - 首次绑定微信奖励 out.remark = `${i18n.t('firstBindingWechatRewards')}`; break; - case 35: // 每日运行云脑任务 - 创建了(CPU/GPU/NPU)类型(调试/训练/推理/评测)任务tangl202204131431995 + case 'CreateCloudbrainTask': // 每日运行云脑任务 - 创建了(CPU/GPU/NPU)类型(调试/训练/推理/评测)任务tangl202204131431995 out.remark = `${i18n.t('created')}${record.Action?.Cloudbrain?.ComputeResource}${i18n.t('type')}${getJobType(record.Action?.Cloudbrain?.JobType)} ${record.Action.RefName}`; break; - case 36: // 数据集被平台推荐 - 数据集XXX被设置为推荐数据集 + case 'DatasetRecommended': // 数据集被平台推荐 - 数据集XXX被设置为推荐数据集 out.remark = `${i18n.t('dataset')}${record.Action.Content && record.Action.Content.split('|')[1]}${i18n.t('setAsRecommendedDataset')}`; break; - case 37: // 提交新公开镜像 - 提交了镜像jiangxiang_ceshi_tang03 + case 'CreateImage': // 提交新公开镜像 - 提交了镜像jiangxiang_ceshi_tang03 out.remark = `${i18n.t('committedImage')}${record.Action.Content && record.Action.Content.split('|')[1]}`; break; - case 38: // 镜像被平台推荐 - 镜像XXX被设置为推荐镜像 + case 'ImageRecommend': // 镜像被平台推荐 - 镜像XXX被设置为推荐镜像 out.remark = `${i18n.t('image')}${record.Action.Content && record.Action.Content.split('|')[1]}${i18n.t('setAsRecommendedImage')}`; break; - case 39: // 首次更换头像 - 更新了头像 + case 'ChangeUserAvatar': // 首次更换头像 - 更新了头像 out.remark = `${i18n.t('updatedAvatar')}`; break; - case 40: // 每日commit - 推送了xxxx分支的代码到OpenI/aiforge - const words = record.Action.RefName.split('/'); - const branch = words[words.length - 1]; - out.remark = `${i18n.t('pushedBranch', { - branch: `${branch}` - })}${record.Action.ShortRepoFullDisplayName}`; + case 'PushCommits': // 每日commit - 推送了xxxx分支的代码到OpenI/aiforge + const opType = record.Action.OpType; + if (opType == 5) { + const words = record.Action.RefName.split('/'); + const branch = words[words.length - 1]; + out.remark = `${i18n.t('pushedBranch', { + branch: `${branch}` + })}${record.Action.ShortRepoFullDisplayName}`; + } else if (opType == 9) { + const words = record.Action.RefName.split('/'); + const tag = words[words.length - 1]; + out.remark = `${i18n.t('pushedTag', { + tag: `${tag}` + })}${record.Action.ShortRepoFullDisplayName}`; + } else if (opType == 16) { + const words = record.Action.RefName.split('/'); + const tag = words[words.length - 1]; + out.remark = `${i18n.t('deleteTag', { + repo: `${record.Action.ShortRepoFullDisplayName}`, + tag: tag, + })}`; + } else if (opType == 17) { + const words = record.Action.RefName.split('/'); + const branch = words[words.length - 1]; + out.remark = `${i18n.t('deleteBranch', { + repo: `${record.Action.ShortRepoFullDisplayName}`, + branch: branch, + })}`; + } break; default: break; @@ -147,13 +170,7 @@ export const getRewardPointRecordInfo = (record) => { } else if (record.SourceType === 'ACCOMPLISH_TASK') { // } else if (record.SourceType === 'RUN_CLOUDBRAIN_TASK') { - out.taskName = `${record?.Cloudbrain?.DisplayJobName}`; - // if (record?.Cloudbrain?.ComputeResource === 'CPU/GPU') { - // const resourceSpec = record?.Cloudbrain?.ResourceSpec?.ResourceSpec; - // out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${record?.Cloudbrain?.ComputeResource}】【GPU: ${resourceSpec?.gpu}, CPU: ${resourceSpec?.cpu}, ${i18n.t('memory')}: ${(resourceSpec?.memMiB / 1024).toFixed(2)}GB, ${i18n.t('sharedMemory')}: ${(resourceSpec?.shareMemMiB / 1024).toFixed(2)}GB】`; - // } else { - // out.remark = `【${getJobType(record?.Cloudbrain?.JobType)}】【${record?.Cloudbrain?.ComputeResource}】【${record?.Cloudbrain?.ResourceSpec.FlavorInfo.desc}】`; - // } + out.taskName = `${record?.Cloudbrain?.DisplayJobName}`; const resourceSpec = record?.Cloudbrain?.ResourceSpec; if (resourceSpec) { resourceSpec.workServerNumber = record?.Cloudbrain?.WorkServerNumber || 1; From 2e23b156f11b6936fe693c195004fb98d703fe6e Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 21 Sep 2022 18:13:52 +0800 Subject: [PATCH 08/24] #1249 update --- models/repo_watch.go | 9 +++++++++ models/task_config.go | 2 +- modules/notification/action/action.go | 22 +++++++--------------- services/task/task.go | 35 ++++++++++++++++++++++++++++++++--- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/models/repo_watch.go b/models/repo_watch.go index 485874301..573a2d78a 100644 --- a/models/repo_watch.go +++ b/models/repo_watch.go @@ -183,6 +183,7 @@ func notifyWatchers(e Engine, actions ...*Action) error { var permCode []bool var permIssue []bool var permPR []bool + var permDataset []bool for _, act := range actions { repoChanged := repo == nil || repo.ID != act.RepoID @@ -234,12 +235,14 @@ func notifyWatchers(e Engine, actions ...*Action) error { permCode = make([]bool, len(watchers)) permIssue = make([]bool, len(watchers)) permPR = make([]bool, len(watchers)) + permDataset = make([]bool, len(watchers)) for i, watcher := range watchers { user, err := getUserByID(e, watcher.UserID) if err != nil { permCode[i] = false permIssue[i] = false permPR[i] = false + permDataset[i] = false continue } perm, err := getUserRepoPermission(e, repo, user) @@ -247,11 +250,13 @@ func notifyWatchers(e Engine, actions ...*Action) error { permCode[i] = false permIssue[i] = false permPR[i] = false + permDataset[i] = false continue } permCode[i] = perm.CanRead(UnitTypeCode) permIssue[i] = perm.CanRead(UnitTypeIssues) permPR[i] = perm.CanRead(UnitTypePullRequests) + permDataset[i] = perm.CanRead(UnitTypeDatasets) } } @@ -276,6 +281,10 @@ func notifyWatchers(e Engine, actions ...*Action) error { if !permPR[i] { continue } + case ActionDatasetRecommended: + if !permDataset[i] { + continue + } } if _, err = e.InsertOne(act); err != nil { diff --git a/models/task_config.go b/models/task_config.go index 8650851be..0d9d21187 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -21,7 +21,7 @@ const ( TaskCreateNewModelTask TaskType = "CreateNewModelTask" TaskBindWechat TaskType = "BindWechat" TaskCreateCloudbrainTask TaskType = "CreateCloudbrainTask" - TaskDatasetRecommended TaskType = "DatasetRecommended " + TaskDatasetRecommended TaskType = "DatasetRecommended" TaskCreateImage TaskType = "CreateImage" TaskImageRecommend TaskType = "ImageRecommend" TaskChangeUserAvatar TaskType = "ChangeUserAvatar" diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 84181c555..5142369e3 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -363,22 +363,14 @@ func (t *actionNotifier) NotifyWechatBind(user *models.User, wechatOpenId string func (t *actionNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *models.Dataset, action string) { switch action { case "recommend": - users, err := models.GetAllDatasetContributorByDatasetId(dataset.ID) - if err != nil { - return - } - var actions = make([]*models.Action, 0) - for _, user := range users { - actions = append(actions, &models.Action{ - OpType: models.ActionDatasetRecommended, - ActUserID: user.ID, - ActUser: user, - RepoID: dataset.RepoID, - Repo: dataset.Repo, - Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), - }) + act := &models.Action{ + OpType: models.ActionDatasetRecommended, + ActUserID: dataset.UserID, + RepoID: dataset.RepoID, + Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), } - if err := models.NotifyWatchers(actions...); err != nil { + + if err := models.NotifyWatchers(act); err != nil { log.Error("notifyWatchers: %v", err) } } diff --git a/services/task/task.go b/services/task/task.go index 5a2028572..ed320a3c5 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -6,6 +6,8 @@ import ( "code.gitea.io/gitea/services/reward" "code.gitea.io/gitea/services/reward/limiter" "fmt" + "strconv" + "strings" ) func Accomplish(action models.Action) { @@ -20,7 +22,8 @@ func Accomplish(action models.Action) { log.Info("Accomplish finished.taskType is not exist.action.ID=%d", action.ID) return } - + actions := make([]models.Action, 0) + actions = append(actions, action) switch taskType { //only creating public repo can be rewarded case models.TaskCreatePublicRepo: @@ -43,9 +46,35 @@ func Accomplish(action models.Action) { log.Debug("the wechat account has been bound before,wechatOpenId = %s", action.Content) return } + case models.TaskDatasetRecommended: + datasetIdStr := strings.Split(action.Content, "|")[0] + datasetId, _ := strconv.ParseInt(datasetIdStr, 10, 64) + users, err := models.GetAllDatasetContributorByDatasetId(datasetId) + if err != nil { + return + } + for _, user := range users { + if user.ID == action.ActUserID { + continue + } + actions = append(actions, models.Action{ + ID: action.ID, + OpType: models.ActionDatasetRecommended, + ActUserID: action.UserID, + UserID: user.ID, + RepoID: action.RepoID, + Content: action.Content, + }) + } + + } + batchAccomplish(taskType, actions...) +} +func batchAccomplish(taskType models.TaskType, actions ...models.Action) { + for _, act := range actions { + go accomplish(act, taskType) } - go accomplish(action, taskType) } func accomplish(action models.Action, taskType models.TaskType) error { @@ -55,7 +84,7 @@ func accomplish(action models.Action, taskType models.TaskType) error { log.Error("PANIC:%v", combinedErr) } }() - userId := action.ActUserID + userId := action.UserID //get task config config, err := GetTaskConfig(string(taskType)) From febd5f43598ef1ee6cc2751bdb199140eeb5f959 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 22 Sep 2022 15:56:36 +0800 Subject: [PATCH 09/24] #1249 update --- modules/notification/action/action.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/notification/action/action.go b/modules/notification/action/action.go index 5142369e3..b1e0ec018 100644 --- a/modules/notification/action/action.go +++ b/modules/notification/action/action.go @@ -367,6 +367,7 @@ func (t *actionNotifier) NotifyDatasetRecommend(optUser *models.User, dataset *m OpType: models.ActionDatasetRecommended, ActUserID: dataset.UserID, RepoID: dataset.RepoID, + IsPrivate: false, Content: fmt.Sprintf("%d|%s", dataset.ID, dataset.Title), } From cbc9ddd9edaa8df2c26695c7c9fe4ff339b5b6a1 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Thu, 22 Sep 2022 16:23:57 +0800 Subject: [PATCH 10/24] #1249 update --- services/socketwrap/clientManager.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 98bcc8a85..7470b1198 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35} type ClientsManager struct { Clients *orderedmap.OrderedMap @@ -107,16 +107,18 @@ func initActionQueue() { func filterUserPrivateInfo(action *models.Action) { action.Comment = nil - action.ActUser.Email = "" - action.ActUser.Passwd = "" - action.ActUser.PasswdHashAlgo = "" - action.ActUser.PrivateKey = "" - action.ActUser.PublicKey = "" - action.ActUser.Salt = "" - action.ActUser.FullName = "" - action.ActUser.AvatarEmail = "" - action.ActUser.IsAdmin = false - action.ActUser.EmailNotificationsPreference = "" - action.ActUser.IsOperator = false + if action.ActUser != nil { + action.ActUser.Email = "" + action.ActUser.Passwd = "" + action.ActUser.PasswdHashAlgo = "" + action.ActUser.PrivateKey = "" + action.ActUser.PublicKey = "" + action.ActUser.Salt = "" + action.ActUser.FullName = "" + action.ActUser.AvatarEmail = "" + action.ActUser.IsAdmin = false + action.ActUser.EmailNotificationsPreference = "" + action.ActUser.IsOperator = false + } } From 0b47b007ddad02a3804bbbacceec64546ebf10ff Mon Sep 17 00:00:00 2001 From: chenshihai Date: Thu, 22 Sep 2022 16:42:43 +0800 Subject: [PATCH 11/24] update activity --- public/home/home.js | 14 ++------------ templates/user/dashboard/feeds.tmpl | 12 +++--------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/public/home/home.js b/public/home/home.js index 4127362c0..c1849b3e3 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -164,18 +164,8 @@ document.onreadystatechange = function () { html += " " + record.RefName + "" } else if(record.OpType == "35"){ - var datasetLink = ''; - actionName.replace('{dataset}', datasetLink); - html += recordPrefix + actionName; - } - else if(record.OpType == "36"){ - var imageLink = ''; - actionName.replace('{image}', imageLink); - html += recordPrefix + actionName; - } - else if(record.OpType == "37"){ - var imageLink = ''; - actionName.replace('{image}', imageLink); + var datasetLink = "" + record.Content.split('|')[1] + ""; + actionName = actionName.replace('{dataset}', datasetLink); html += recordPrefix + actionName; } else{ diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index f3ec10c4a..c2da66ff5 100755 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -90,16 +90,8 @@ {{$.i18n.Tr "action.task_c2netnputrainjob" .GetRepoLink .Content .RefName | Str2html}} {{else if eq .GetOpType 33}} {{$.i18n.Tr "action.task_c2netgputrainjob" .GetRepoLink .Content .RefName | Str2html}} - {{else if eq .GetOpType 34}} - {{$.i18n.Tr "action.binded_wechat"}} {{else if eq .GetOpType 35}} - {{$.i18n.Tr "action.dataset_recommended" .GetRepoLink | Str2html}} - {{else if eq .GetOpType 36}} - {{$.i18n.Tr "action.create_image" .GetRepoLink | Str2html}} - {{else if eq .GetOpType 37}} - {{$.i18n.Tr "action.image_recommended" .GetRepoLink | Str2html}} - {{else if eq .GetOpType 38}} - {{$.i18n.Tr "action.update_user_avatar"}} + {{$.i18n.Tr "action.dataset_recommended" .GetRepoLink (index .GetIssueInfos 1 | RenderEmoji) | Str2html}} {{end}}

{{if or (eq .GetOpType 5) (eq .GetOpType 18)}} @@ -147,6 +139,8 @@ {{else if eq .GetOpType 31}} + {{else if eq .GetOpType 35}} + {{else}} {{svg (printf "octicon-%s" (ActionIcon .GetOpType)) 32}} {{end}} From 319231d579a8f67571d6950406727ac0b1974d54 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 22 Sep 2022 21:30:47 +0800 Subject: [PATCH 12/24] grampus-storage --- models/cloudbrain.go | 27 +++++++++++++++++---------- modules/grampus/grampus.go | 38 +++++++++++++++++++++++++++++++++++++- routers/repo/grampus.go | 18 ++++++++++-------- 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c05ba0f21..e64543722 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -657,8 +657,6 @@ type FlavorInfo struct { UnitPrice int64 `json:"unitPrice"` } - - type SpecialPools struct { Pools []*SpecialPool `json:"pools"` } @@ -1476,14 +1474,23 @@ type GrampusStopJobResponse struct { } type GrampusTasks struct { - Command string `json:"command"` - Name string `json:"name"` - ImageId string `json:"imageId"` - ResourceSpecId string `json:"resourceSpecId"` - ImageUrl string `json:"imageUrl"` - CenterID []string `json:"centerID"` - CenterName []string `json:"centerName"` - ReplicaNum int `json:"replicaNum"` + Command string `json:"command"` + Name string `json:"name"` + ImageId string `json:"imageId"` + ResourceSpecId string `json:"resourceSpecId"` + ImageUrl string `json:"imageUrl"` + CenterID []string `json:"centerID"` + CenterName []string `json:"centerName"` + ReplicaNum int `json:"replicaNum"` + Datasets []GrampusDataset `json:"datasets"` + Models []GrampusDataset `json:"models"` +} + +type GrampusDataset struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + EndPoint string `json:"endPoint"` + ObjectKey string `json:"objectKey"` } type CreateGrampusJobRequest struct { diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index 83fc3b1d4..d72a7b10e 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -75,12 +75,46 @@ type GenerateTrainJobReq struct { Spec *models.Specification } +func getEndPoint() string { + index := strings.Index(setting.Endpoint, "//") + endpoint := setting.Endpoint[index+2:] + return endpoint +} + +func getDatasetGrampus(datasetInfos map[string]models.DatasetInfo) []models.GrampusDataset { + var datasetGrampus []models.GrampusDataset + endPoint := getEndPoint() + for _, datasetInfo := range datasetInfos { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: datasetInfo.FullName, + Bucket: setting.Bucket, + EndPoint: endPoint, + ObjectKey: datasetInfo.DataLocalPath + datasetInfo.FullName, + }) + + } + return datasetGrampus +} + func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { createTime := timeutil.TimeStampNow() centerID, centerName := getCentersParamter(ctx, req) - log.Info("grampus Command:" + req.Command) + var datasetGrampus, modelGrampus []models.GrampusDataset + if ProcessorTypeNPU == req.ProcessType { + datasetGrampus = getDatasetGrampus(req.DatasetInfos) + if len(req.ModelName) != 0 { + modelGrampus = []models.GrampusDataset{ + { + Name: req.ModelName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.PreTrainModelPath, + }, + } + } + } jobResult, err := createJob(models.CreateGrampusJobRequest{ Name: req.JobName, @@ -94,6 +128,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error CenterID: centerID, CenterName: centerName, ReplicaNum: 1, + Datasets: datasetGrampus, + Models: modelGrampus, }, }, }) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index dc05690df..e73742515 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/services/reward/point/account" - "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/grampus" @@ -721,7 +720,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion req.PreTrainModelUrl = form.PreTrainModelUrl - + req.PreTrainModelPath = preTrainModelPath } err = grampus.GenerateTrainJob(ctx, req) @@ -950,8 +949,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" - commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) + commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" command += commandDownload } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" @@ -960,10 +958,14 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } //unzip code & dataset - unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) - - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand - command += commandUnzip + if processorType == grampus.ProcessorTypeNPU { + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" + command += commandUnzip + } else if processorType == grampus.ProcessorTypeGPU { + unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand + command += commandUnzip + } command += "echo \"unzip finished;start to exec code;\";" From 5d7ed8006aebe836b9e35ad067d42ca5fa753766 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 23 Sep 2022 12:16:31 +0800 Subject: [PATCH 13/24] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 2 +- templates/repo/modelarts/trainjob/version_new.tmpl | 8 ++- web_src/js/features/cloudbrainShow.js | 68 +++++++++++++++++++--- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 125d11517..f6a086f40 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -256,7 +256,7 @@ {{if .CanModify}} {{$.i18n.Tr "repo.modelarts.modify"}} + href="{{$.RepoLink}}/modelarts/train-job/{{.JobID}}/create_version?version_name={{.VersionName}}&path=show">{{$.i18n.Tr "repo.modelarts.modify"}} {{else}} {{$.i18n.Tr "repo.modelarts.modify"}} diff --git a/templates/repo/modelarts/trainjob/version_new.tmpl b/templates/repo/modelarts/trainjob/version_new.tmpl index 856b307cf..89bf6fc08 100644 --- a/templates/repo/modelarts/trainjob/version_new.tmpl +++ b/templates/repo/modelarts/trainjob/version_new.tmpl @@ -261,7 +261,7 @@ - {{.i18n.Tr "repo.cloudbrain.cancel"}} + {{.i18n.Tr "repo.cloudbrain.cancel"}} @@ -274,7 +274,11 @@