package cloudbrain import ( "io/ioutil" "os" "sort" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) func ClearCloudbrainResultSpace() { log.Info("clear cloudbrain one result space begin.") if !setting.ClearStrategy.Enabled { return } tasks, err := models.GetGPUStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) return } debugTasks, err := models.GetGPUStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize) if err != nil { log.Warn("Failed to get debug cloudbrain.", err) } tasks = append(tasks, debugTasks...) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) return } var ids []int64 for _, task := range tasks { err := DeleteCloudbrainOneJobStorage(task.JobName) if err == nil { log.Info("clear job in cloudbrain table:" + task.JobName) ids = append(ids, task.ID) } } err = models.UpdateCloudBrainRecordsCleared(ids) if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } //如果云脑表处理完了,通过遍历minio对象处理历史垃圾数据,如果存在的话 if len(tasks) < setting.ClearStrategy.BatchSize+setting.ClearStrategy.DebugJobSize { clearLocalHistoryTrashFile() clearMinioHistoryTrashFile() } log.Info("clear cloudbrain one result space end.") } func clearMinioHistoryTrashFile() { JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix miniofiles, err := ioutil.ReadDir(JobRealPrefix) processCount := 0 if err != nil { log.Warn("Can not browser minio job path.") } else { SortModTimeAscend(miniofiles) for _, file := range miniofiles { if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { has, err := models.IsCloudbrainExistByJobName(file.Name()) if err == nil && !has { dirPath := setting.CBCodePathPrefix + file.Name() + "/" log.Info("clear job in minio trash:" + file.Name()) storage.Attachments.DeleteDir(dirPath) processCount++ } if processCount == setting.ClearStrategy.BatchSize { break } } else { break } } } } func clearLocalHistoryTrashFile() { files, err := ioutil.ReadDir(setting.JobPath) processCount := 0 if err != nil { log.Warn("Can not browser local job path.") } else { SortModTimeAscend(files) for _, file := range files { //清理n天前的历史垃圾数据,清理job目录 if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { has, err := models.IsCloudbrainExistByJobName(file.Name()) if err == nil && !has { os.RemoveAll(setting.JobPath + file.Name()) log.Info("clear job in local trash:" + file.Name()) processCount++ } if processCount == setting.ClearStrategy.BatchSize { break } } else { break } } } } func SortModTimeAscend(files []os.FileInfo) { sort.Slice(files, func(i, j int) bool { return files[i].ModTime().Before(files[j].ModTime()) }) } func DeleteCloudbrainOneJobStorage(jobName string) error { if jobName == "" { return nil } //delete local localJobPath := setting.JobPath + jobName err := os.RemoveAll(localJobPath) if err != nil { log.Error("RemoveAll(%s) failed:%v", localJobPath, err) } dirPath := setting.CBCodePathPrefix + jobName + "/" err1 := storage.Attachments.DeleteDir(dirPath) if err1 != nil { log.Error("DeleteDir(%s) failed:%v", localJobPath, err) } if err == nil { err = err1 } return err }