From 394bfdd8a70dee03091d45fda7fbff1b6d3527ca Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 30 Nov 2022 18:06:18 +0800 Subject: [PATCH 01/16] #3169 update --- models/resource_specification.go | 93 ++++++++++++++++++++++ modules/grampus/grampus.go | 5 +- .../cloudbrain/resource/resource_specification.go | 48 +---------- 3 files changed, 96 insertions(+), 50 deletions(-) diff --git a/models/resource_specification.go b/models/resource_specification.go index 2f815818b..809a3496a 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -3,6 +3,7 @@ package models import ( "code.gitea.io/gitea/modules/timeutil" "fmt" + "strings" "xorm.io/builder" ) @@ -197,12 +198,104 @@ type Specification struct { AiCenterName string IsExclusive bool ExclusiveOrg string + //specs that have the same sourceSpecId, computeResource and cluster as current spec + RelatedSpecs []*Specification } func (Specification) TableName() string { return "resource_specification" } +func (s *Specification) loadRelatedSpecs() { + if s.RelatedSpecs != nil { + return + } + defaultSpecs := make([]*Specification, 0) + if s.SourceSpecId == "" { + s.RelatedSpecs = defaultSpecs + return + } + r, err := FindSpecs(FindSpecsOptions{ + ComputeResource: s.ComputeResource, + Cluster: s.Cluster, + SourceSpecId: s.SourceSpecId, + RequestAll: true, + SpecStatus: SpecOnShelf, + }) + if err != nil { + s.RelatedSpecs = defaultSpecs + return + } + s.RelatedSpecs = r +} +func (s *Specification) GetAvailableCenterIds(userIds ...int64) []string { + s.loadRelatedSpecs() + + if len(s.RelatedSpecs) == 0 { + return make([]string, 0) + } + + var uId int64 + if len(userIds) > 0 { + uId = userIds[0] + } + //filter exclusive specs + specs := FilterExclusiveSpecs(s.RelatedSpecs, uId) + + centerIds := make([]string, len(specs)) + for i, v := range specs { + centerIds[i] = v.AiCenterCode + } + return centerIds +} + +func FilterExclusiveSpecs(r []*Specification, userId int64) []*Specification { + if userId == 0 { + return r + } + specs := make([]*Specification, 0, len(r)) + specMap := make(map[int64]string, 0) + for i := 0; i < len(r); i++ { + spec := r[i] + if _, has := specMap[spec.ID]; has { + continue + } + if !spec.IsExclusive { + specs = append(specs, spec) + specMap[spec.ID] = "" + continue + } + orgs := strings.Split(spec.ExclusiveOrg, ";") + for _, org := range orgs { + isMember, _ := IsOrganizationMemberByOrgName(org, userId) + if isMember { + specs = append(specs, spec) + specMap[spec.ID] = "" + break + } + } + } + return specs +} + +func DistinctSpecs(r []*Specification) []*Specification { + specs := make([]*Specification, 0, len(r)) + sourceSpecIdMap := make(map[string]string, 0) + for i := 0; i < len(r); i++ { + spec := r[i] + if spec.SourceSpecId == "" { + specs = append(specs, spec) + continue + } + if _, has := sourceSpecIdMap[spec.SourceSpecId]; has { + continue + } + specs = append(specs, spec) + sourceSpecIdMap[spec.SourceSpecId] = "" + } + return specs +} + func InsertResourceSpecification(r ResourceSpecification) (int64, error) { return x.Insert(&r) } diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index b6f62560a..6b2ea6288 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -105,8 +105,6 @@ func getDatasetGrampus(datasetInfos map[string]models.DatasetInfo) []models.Gram func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { createTime := timeutil.TimeStampNow() - centerID, centerName := getCentersParamter(ctx, req) - var datasetGrampus, modelGrampus []models.GrampusDataset var codeGrampus models.GrampusDataset if ProcessorTypeNPU == req.ProcessType { @@ -138,8 +136,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str ResourceSpecId: req.Spec.SourceSpecId, ImageId: req.ImageId, ImageUrl: req.ImageUrl, - CenterID: centerID, - CenterName: centerName, + CenterID: req.Spec.GetAvailableCenterIds(ctx.User.ID), ReplicaNum: 1, Datasets: datasetGrampus, Models: modelGrampus, diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index 8f4182d87..5070d7c1e 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -246,10 +246,10 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S return nil, err } //filter exclusive specs - specs := filterExclusiveSpecs(r, userId) + specs := models.FilterExclusiveSpecs(r, userId) //distinct by sourceSpecId - specs = distinctSpecs(specs) + specs = models.DistinctSpecs(specs) return specs, err } @@ -265,50 +265,6 @@ func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*api return result, nil } -func filterExclusiveSpecs(r []*models.Specification, userId int64) []*models.Specification { - specs := make([]*models.Specification, 0, len(r)) - specMap := make(map[int64]string, 0) - for i := 0; i < len(r); i++ { - spec := r[i] - if _, has := specMap[spec.ID]; has { - continue - } - if !spec.IsExclusive { - specs = append(specs, spec) - specMap[spec.ID] = "" - continue - } - orgs := strings.Split(spec.ExclusiveOrg, ";") - for _, org := range orgs { - isMember, _ := models.IsOrganizationMemberByOrgName(org, userId) - if isMember { - specs = append(specs, spec) - specMap[spec.ID] = "" - break - } - } - } - return specs -} - -func distinctSpecs(r []*models.Specification) []*models.Specification { - specs := make([]*models.Specification, 0, len(r)) - sourceSpecIdMap := make(map[string]string, 0) - for i := 0; i < len(r); i++ { - spec := r[i] - if spec.SourceSpecId == "" { - specs = append(specs, spec) - continue - } - if _, has := sourceSpecIdMap[spec.SourceSpecId]; has { - continue - } - specs = append(specs, spec) - sourceSpecIdMap[spec.SourceSpecId] = "" - } - return specs -} - func GetAndCheckSpec(userId int64, specId int64, opts models.FindSpecsOptions) (*models.Specification, error) { if specId == 0 { return nil, nil From 85d03023bd7baaa53f3e8addad8368e758313585 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 2 Dec 2022 11:13:09 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cron/tasks_basic.go | 17 ++++++- modules/setting/setting.go | 5 ++ services/cloudbrain/clear.go | 108 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 services/cloudbrain/clear.go diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 985a82cdb..9b74c1637 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,10 +5,12 @@ package cron import ( - "code.gitea.io/gitea/modules/urfs_client/urchin" "context" "time" + "code.gitea.io/gitea/modules/urfs_client/urchin" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward" @@ -190,6 +192,17 @@ func registerHandleRepoAndUserStatistic() { }) } +func registerHandleClearCloudbrainResult() { + RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "* 0,30 2-8 * * ? *", + }, func(ctx context.Context, _ *models.User, _ Config) error { + cloudbrainService.ClearCloudbrainResultSpace() + return nil + }) +} + func registerHandleSummaryStatistic() { RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ Enabled: true, @@ -317,6 +330,6 @@ func initBasicTasks() { registerHandleModelSafetyTask() - registerHandleScheduleRecord() + registerHandleScheduleRecord() registerHandleCloudbrainDurationStatistic() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a473dad6a..2b2ded678 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,6 +518,8 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string + ResultSaveDays int64 + ResultBatchSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,6 +1482,9 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) + ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) + sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go new file mode 100644 index 000000000..52e5c5cf1 --- /dev/null +++ b/services/cloudbrain/clear.go @@ -0,0 +1,108 @@ +package cloudbrain + +import ( + "io/ioutil" + "os" + "sort" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +func ClearCloudbrainResultSpace() { + + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + + if err != nil { + log.Warn("Failed to get cloudbrain, clear result failed.", err) + return + } + var ids []int64 + for _, task := range tasks { + err := DeleteCloudbrainOneJobStorage(task.JobName) + if err == nil { + ids = append(ids, task.ID) + } + } + + err = models.UpdateCloudBrainRecordsCleared(ids) + if err != nil { + log.Warn("Failed to set cloudbrain cleared status", err) + } + if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + files, err := ioutil.ReadDir(setting.JobPath) + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理4个月前的任务 + if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { + //os.RemoveAll(setting.JobPath + file.Name()) + } else { + break + } + } + + } + + minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) + + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscendForMinio(minioFiles) + for _, file := range minioFiles { + //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) + if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + dirPath := setting.CBCodePathPrefix + file.FileName + "/" + log.Info(dirPath) + //storage.Attachments.DeleteDir(dirPath) + } else { + break + } + } + + } + + } + +} + +func SortModTimeAscend(files []os.FileInfo) { + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime().Before(files[j].ModTime()) + }) +} +func SortModTimeAscendForMinio(files []storage.FileInfo) { + sort.Slice(files, func(i, j int) bool { + timeI, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + timeJ, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + return timeI.Before(timeJ) + }) +} + +func DeleteCloudbrainOneJobStorage(jobName string) error { + //delete local + localJobPath := setting.JobPath + jobName + err := os.RemoveAll(localJobPath) + if err != nil { + log.Error("RemoveAll(%s) failed:%v", localJobPath, err) + } + + dirPath := setting.CBCodePathPrefix + jobName + "/" + err1 := storage.Attachments.DeleteDir(dirPath) + + if err1 != nil { + log.Error("DeleteDir(%s) failed:%v", localJobPath, err) + } + if err == nil { + err = err1 + } + + return err +} From 9b565469d1d02ce020cfd4a7a36159ae32080198 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 2 Dec 2022 11:19:55 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1e7c702ab..e492d9eb9 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2050,6 +2050,64 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } +func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0, 10) + endTimeBefore := time.Now().Unix() - days*24*3600 + missEndTimeBefore := endTimeBefore - 24*3600 + return cloudbrains, x.Cols("id,job_name,job_id"). + In("status", + JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, + ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, + ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). + Where("((end_time is null and created_unix 0 { + idsIn := "" + for i, id := range tempIds { + if i == 0 { + idsIn += strconv.FormatInt(id, 10) + } else { + idsIn += "," + strconv.FormatInt(id, 10) + } + } + _, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids) + if errTemp != nil { + err = errTemp + } + + } + + } + return err + +} + +func getPageIds(ids []int64, page int, pagesize int) []int64 { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(ids)-1 { + return []int64{} + } + if end > len(ids)-1 { + return ids[begin:] + } else { + return ids[begin:end] + } + +} + func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0) return cloudbrains, x. From 0111ef5d923ccbe1a46dcb0bff30a606ccb734eb Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 6 Dec 2022 08:52:00 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 ++-- modules/cron/tasks_basic.go | 4 +++- modules/setting/setting.go | 25 ++++++++++++++++++++----- services/cloudbrain/clear.go | 29 +++++++++++++++++------------ 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e492d9eb9..abb804eef 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2050,9 +2050,9 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) { +func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) - endTimeBefore := time.Now().Unix() - days*24*3600 + endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 return cloudbrains, x.Cols("id,job_name,job_id"). In("status", diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 9b74c1637..1813b1d50 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,6 +5,7 @@ package cron import ( + "code.gitea.io/gitea/modules/setting" "context" "time" @@ -196,7 +197,7 @@ func registerHandleClearCloudbrainResult() { RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "* 0,30 2-8 * * ? *", + Schedule: setting.ClearStrategy.Cron, }, func(ctx context.Context, _ *models.User, _ Config) error { cloudbrainService.ClearCloudbrainResultSpace() return nil @@ -319,6 +320,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleClearCloudbrainResult() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2b2ded678..b5ec488a8 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,8 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - ResultSaveDays int64 - ResultBatchSize int + //benchmark config IsBenchmarkEnabled bool @@ -615,6 +614,15 @@ var ( UsageRateBeginTime string }{} + ClearStrategy= struct { + Enabled bool + ResultSaveDays int + ResultBatchSize int + BatchSize int + TrashSaveDays int + Cron string + }{} + C2NetInfos *C2NetSqInfos CenterInfos *AiCenterInfos C2NetMapInfo map[string]*C2NetSequenceInfo @@ -1482,9 +1490,6 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) - ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) - sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") @@ -1684,6 +1689,16 @@ func getModelartsCDConfig() { getNotebookFlavorInfos() } +func getClearStrategy(){ + + sec := Cfg.Section("clear_strategy") + ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) + ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) + ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) + ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) + ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") +} + func getGrampusConfig() { sec := Cfg.Section("grampus") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 52e5c5cf1..f8dfae30e 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -14,7 +14,7 @@ import ( func ClearCloudbrainResultSpace() { - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,17 +32,20 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 files, err := ioutil.ReadDir(setting.JobPath) + processCount:=0 if err != nil { log.Warn("Can not browser local job path.") } else { SortModTimeAscend(files) for _, file := range files { - //清理4个月前的任务 - if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { - //os.RemoveAll(setting.JobPath + file.Name()) - } else { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + } + if processCount==setting.ClearStrategy.ResultBatchSize{ break } } @@ -50,20 +53,22 @@ func ClearCloudbrainResultSpace() { } minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - + processCount=0 if err != nil { log.Warn("Can not browser minio job path.") } else { SortModTimeAscendForMinio(minioFiles) for _, file := range minioFiles { - //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { dirPath := setting.CBCodePathPrefix + file.FileName + "/" log.Info(dirPath) - //storage.Attachments.DeleteDir(dirPath) - } else { - break + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount==setting.ClearStrategy.ResultBatchSize{ + break + } } } From bd77ae4573cf0834f274bfa8e40b413e2137c59a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 7 Dec 2022 16:18:21 +0800 Subject: [PATCH 05/16] fix-3315 --- routers/api/v1/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8958c55a4..c18a290a6 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -738,7 +738,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken(), repoAssignment()) m.Group("/file_notebook", func() { - m.Get("", reqToken(), repo.GetFileNoteBookInfo) + m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) }) From a56551d5e50bce11116003a6c1513e5a1cae4d78 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 7 Dec 2022 17:25:22 +0800 Subject: [PATCH 06/16] fix-3255 --- routers/api/v1/repo/cloudbrain_dashboard.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7fe5d603c..0d68fff30 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -968,6 +968,8 @@ func GetWaittingTop(ctx *context.Context) { taskDetail.RepoID = ciTasks[i].RepoID if ciTasks[i].Repo != nil { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name + } else { + taskDetail.RepoName = "" } WaitTimeInt := time.Now().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) @@ -975,6 +977,13 @@ func GetWaittingTop(ctx *context.Context) { if WaitTimeInt < 0 { taskDetail.WaitTime = "00:00:00" } + + taskDetail.ID = ciTasks[i].Cloudbrain.ID + taskDetail.ComputeResource = ciTasks[i].Cloudbrain.ComputeResource + taskDetail.JobType = ciTasks[i].Cloudbrain.JobType + taskDetail.JobID = ciTasks[i].Cloudbrain.JobID + taskDetail.Type = ciTasks[i].Cloudbrain.Type + tasks = append(tasks, taskDetail) } ctx.JSON(http.StatusOK, map[string]interface{}{ @@ -1001,6 +1010,12 @@ func GetRunningTop(ctx *context.Context) { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name } + taskDetail.ID = ciTasks[i].Cloudbrain.ID + taskDetail.ComputeResource = ciTasks[i].Cloudbrain.ComputeResource + taskDetail.JobType = ciTasks[i].Cloudbrain.JobType + taskDetail.JobID = ciTasks[i].Cloudbrain.JobID + taskDetail.Type = ciTasks[i].Cloudbrain.Type + tasks = append(tasks, taskDetail) } ctx.JSON(http.StatusOK, map[string]interface{}{ From 47913c8be53cfee7b07e75403203c5445e69adc6 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 7 Dec 2022 17:44:26 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E6=B8=85=E7=90=86=E4=BA=91=E8=84=91?= =?UTF-8?q?=E4=B8=80=E5=8E=86=E5=8F=B2=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 +- modules/setting/setting.go | 4 +- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 2 + routers/repo/cloudbrain.go | 12 ++++++ services/cloudbrain/clear.go | 83 +++++++++++++++++++++++++---------------- 6 files changed, 72 insertions(+), 34 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index abb804eef..2de66a834 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -204,6 +204,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -2081,7 +2082,8 @@ func UpdateCloudBrainRecordsCleared(ids []int64) error { idsIn += "," + strconv.FormatInt(id, 10) } } - _, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids) + + _, errTemp := x.Unscoped().Exec("update cloudbrain set cleared=true where id in (" + idsIn + ")") if errTemp != nil { err = errTemp } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b5ec488a8..8815c6dcf 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -617,10 +617,10 @@ var ( ClearStrategy= struct { Enabled bool ResultSaveDays int - ResultBatchSize int BatchSize int TrashSaveDays int Cron string + RunAtStart bool }{} C2NetInfos *C2NetSqInfos @@ -1629,6 +1629,7 @@ func NewContext() { getModelConvertConfig() getModelSafetyConfig() getModelAppConfig() + getClearStrategy() } func getModelSafetyConfig() { @@ -1697,6 +1698,7 @@ func getClearStrategy(){ ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") + ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) } func getGrampusConfig() { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 647bdb1ad..1ac1a0bd7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3246,6 +3246,7 @@ specification = specification select_specification = select specification description = description wrong_specification=You cannot use this specification, please choose another item. +result_cleared=The files of the task have been cleared, can not restart any more, please create a new debug task instead. resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8f9e6b664..792981c86 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3266,6 +3266,8 @@ card_duration = 运行卡时 card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 +result_cleared=本任务的文件已被清理，无法再次调试，请新建调试任务。 + job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d3d76f440..a23cd5462 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -670,6 +670,13 @@ func CloudBrainRestart(ctx *context.Context) { break } + if _, err := os.Stat(getOldJobPath(task)); err != nil { + log.Error("Can not find job minio path", err) + resultCode = "-1" + errorMsg = ctx.Tr("cloudbrain.result_cleared") + break + } + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -704,6 +711,11 @@ func CloudBrainRestart(ctx *context.Context) { }) } + +func getOldJobPath(task *models.Cloudbrain) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName +} + func CloudBrainBenchMarkShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark) } diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index f8dfae30e..2cefb8341 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -13,8 +13,11 @@ import ( ) func ClearCloudbrainResultSpace() { + if !setting.ClearStrategy.Enabled{ + return + } - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,44 +35,60 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 - files, err := ioutil.ReadDir(setting.JobPath) - processCount:=0 - if err != nil { - log.Warn("Can not browser local job path.") - } else { - SortModTimeAscend(files) - for _, file := range files { - //清理n天前的历史垃圾数据，清理job目录 - if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - os.RemoveAll(setting.JobPath + file.Name()) - processCount++ - } - if processCount==setting.ClearStrategy.ResultBatchSize{ + //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话 + if len(tasks) < setting.ClearStrategy.BatchSize { + clearLocalHistoryTrashFile() + clearMinioHistoryTrashFile() + + } + +} + +func clearMinioHistoryTrashFile() { + JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + + miniofiles, err := ioutil.ReadDir(JobRealPrefix) + + processCount := 0 + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscend(miniofiles) + for _, file := range miniofiles { + + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + dirPath := setting.CBCodePathPrefix + file.Name() + "/" + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { break } + } else { + break } } - minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - processCount=0 - if err != nil { - log.Warn("Can not browser minio job path.") - } else { - SortModTimeAscendForMinio(minioFiles) - for _, file := range minioFiles { - - timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - dirPath := setting.CBCodePathPrefix + file.FileName + "/" - log.Info(dirPath) - storage.Attachments.DeleteDir(dirPath) - processCount++ - if processCount==setting.ClearStrategy.ResultBatchSize{ - break - } + } +} + +func clearLocalHistoryTrashFile() { + files, err := ioutil.ReadDir(setting.JobPath) + processCount := 0 + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { + break } + } else { + break } } From f9c9b888eb7435619b1cc01d12d837255977425a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 7 Dec 2022 17:46:13 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cron/tasks_basic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 1813b1d50..6a1fc6e39 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -196,7 +196,7 @@ func registerHandleRepoAndUserStatistic() { func registerHandleClearCloudbrainResult() { RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ Enabled: true, - RunAtStart: false, + RunAtStart: setting.ClearStrategy.RunAtStart, Schedule: setting.ClearStrategy.Cron, }, func(ctx context.Context, _ *models.User, _ Config) error { cloudbrainService.ClearCloudbrainResultSpace() From 516ba2e108c19a53b85f0edca8a97e32e012c467 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 7 Dec 2022 17:53:02 +0800 Subject: [PATCH 09/16] fix-3255 --- models/cloudbrain_static.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index a213f179c..40d7a2a2e 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -183,6 +183,17 @@ func GetWaittingTop() ([]*CloudbrainInfo, error) { Find(&cloudbrains); err != nil { log.Info("find error.") } + + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) + if err == nil { + for _, task := range cloudbrains { + task.Repo = repositoryMap[task.RepoID] + } + } return cloudbrains, nil } @@ -199,6 +210,16 @@ func GetRunningTop() ([]*CloudbrainInfo, error) { Find(&cloudbrains); err != nil { log.Info("find error.") } + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) + if err == nil { + for _, task := range cloudbrains { + task.Repo = repositoryMap[task.RepoID] + } + } return cloudbrains, nil } From cb23d9c64b2983c85f19b1bb9f7927f6cd679a3c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 8 Dec 2022 08:57:21 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/clear.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 2cefb8341..bbdfdd024 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -27,6 +27,7 @@ func ClearCloudbrainResultSpace() { for _, task := range tasks { err := DeleteCloudbrainOneJobStorage(task.JobName) if err == nil { + log.Info("clear job in cloudbrain table:"+task.JobName) ids = append(ids, task.ID) } } @@ -58,6 +59,7 @@ func clearMinioHistoryTrashFile() { if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { dirPath := setting.CBCodePathPrefix + file.Name() + "/" + log.Info("clear job in minio trash:"+file.Name()) storage.Attachments.DeleteDir(dirPath) processCount++ if processCount == setting.ClearStrategy.BatchSize { @@ -83,6 +85,7 @@ func clearLocalHistoryTrashFile() { //清理n天前的历史垃圾数据，清理job目录 if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { os.RemoveAll(setting.JobPath + file.Name()) + log.Info("clear job in local trash:"+file.Name()) processCount++ if processCount == setting.ClearStrategy.BatchSize { break From 5975fcb173d3a5991ea7ffdca77d38081ef01777 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 8 Dec 2022 10:25:01 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E8=BF=87=E6=BB=A4end=5Ftime=3D0=E7=9A=84?= =?UTF-8?q?=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 2de66a834..9017c2a86 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2060,7 +2060,7 @@ func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, erro JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). - Where("((end_time is null and created_unix Date: Thu, 8 Dec 2022 10:32:26 +0800 Subject: [PATCH 12/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 9017c2a86..5b427c010 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2060,7 +2060,7 @@ func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, erro JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). - Where("((end_time is null and created_unix Date: Thu, 8 Dec 2022 11:09:29 +0800 Subject: [PATCH 13/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 8815c6dcf..3506e2715 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1697,7 +1697,7 @@ func getClearStrategy(){ ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) - ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") + ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?") ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) } From f1728176e000384a56029c0aeac8d3ae12e30032 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 9 Dec 2022 12:38:22 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=B8=85=E7=90=86=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 30 +++++++++++++++++++++++++++--- modules/setting/setting.go | 2 ++ services/cloudbrain/clear.go | 41 ++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 5b427c010..aeed8629c 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1906,6 +1906,12 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } +func IsCloudbrainExistByJobName(jobName string)(bool,error){ + return x.Unscoped().Exist(&Cloudbrain{ + JobName: jobName, + }) +} + func GetCloudbrainByIDWithDeleted(id string) (*Cloudbrain, error) { idInt64, _ := strconv.ParseInt(id, 10, 64) cb := &Cloudbrain{ID: idInt64} @@ -2051,19 +2057,37 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { +func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 - return cloudbrains, x.Cols("id,job_name,job_id"). + return cloudbrains, x.Unscoped().Cols("id,job_name,job_id"). In("status", JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). - Where("(((end_time is null or end_time=0) and updated_unix Date: Fri, 9 Dec 2022 13:00:05 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/clear.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 259ee2592..6172a8203 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -48,7 +48,7 @@ func ClearCloudbrainResultSpace() { log.Warn("Failed to set cloudbrain cleared status", err) } //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话 - if len(tasks) < setting.ClearStrategy.BatchSize { + if len(tasks) < setting.ClearStrategy.BatchSize+setting.ClearStrategy.DebugJobSize { clearLocalHistoryTrashFile() clearMinioHistoryTrashFile() From 2c1ba17372a285d1a4eda1df89255b4afc86a743 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 9 Dec 2022 17:44:33 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/clear.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 6172a8203..44613ee3c 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -69,7 +69,8 @@ func clearMinioHistoryTrashFile() { SortModTimeAscend(miniofiles) for _, file := range miniofiles { - if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + has,err:=models.IsCloudbrainExistByJobName(file.Name()) if err==nil && !has { dirPath := setting.CBCodePathPrefix + file.Name() + "/" @@ -98,7 +99,7 @@ func clearLocalHistoryTrashFile() { SortModTimeAscend(files) for _, file := range files { //清理n天前的历史垃圾数据，清理job目录 - if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { has,err:=models.IsCloudbrainExistByJobName(file.Name()) if err==nil && !has{ os.RemoveAll(setting.JobPath + file.Name()) @@ -125,6 +126,10 @@ func SortModTimeAscend(files []os.FileInfo) { } func DeleteCloudbrainOneJobStorage(jobName string) error { + + if jobName==""{ + return nil + } //delete local localJobPath := setting.JobPath + jobName err := os.RemoveAll(localJobPath)