Browse Source

Merge remote-tracking branch 'origin/V20221228' into gcu

gcu
chenshihai 2 years ago
parent
commit
c8cf6d34b8
35 changed files with 558 additions and 614 deletions
  1. +15
    -0
      models/cloudbrain.go
  2. +64
    -1
      models/cloudbrain_static.go
  3. +171
    -48
      models/user_business_analysis.go
  4. +9
    -0
      models/user_business_struct.go
  5. +3
    -2
      modules/cron/tasks_basic.go
  6. +12
    -0
      modules/modelarts/modelarts.go
  7. +3
    -1
      modules/modelarts/resty.go
  8. +6
    -0
      modules/setting/setting.go
  9. +3
    -3
      options/locale/locale_en-US.ini
  10. +3
    -3
      options/locale/locale_zh-CN.ini
  11. +3
    -2
      public/home/home.js
  12. +55
    -8
      routers/api/v1/repo/cloudbrain.go
  13. +36
    -60
      routers/api/v1/repo/cloudbrain_dashboard.go
  14. +1
    -1
      routers/repo/ai_model_convert.go
  15. +4
    -1
      routers/repo/aisafety.go
  16. +74
    -120
      routers/repo/cloudbrain_statistic.go
  17. +2
    -2
      routers/repo/grampus.go
  18. +4
    -1
      routers/repo/modelarts.go
  19. +12
    -0
      routers/repo/user_data_analysis.go
  20. +1
    -1
      services/cloudbrain/cloudbrainTask/count.go
  21. +1
    -1
      services/socketwrap/clientManager.go
  22. +18
    -25
      templates/admin/cloudbrain/list.tmpl
  23. +3
    -77
      templates/repo/cloudbrain/new.tmpl
  24. +4
    -66
      templates/repo/grampus/notebook/gpu/new.tmpl
  25. +4
    -81
      templates/repo/grampus/notebook/npu/new.tmpl
  26. +1
    -11
      templates/repo/grampus/notebook/show.tmpl
  27. +2
    -2
      templates/repo/grampus/trainjob/gpu/new.tmpl
  28. +2
    -2
      templates/repo/grampus/trainjob/npu/new.tmpl
  29. +2
    -68
      templates/repo/modelarts/notebook/new.tmpl
  30. +8
    -16
      templates/user/dashboard/cloudbrains.tmpl
  31. +7
    -8
      templates/user/dashboard/feeds.tmpl
  32. +4
    -1
      web_src/js/features/cloudbrainShow.js
  33. +4
    -1
      web_src/js/features/cloudrbanin.js
  34. +12
    -0
      web_src/js/standalone/cloudbrainNew.js
  35. +5
    -1
      web_src/vuepages/pages/notebook/debug/index.vue

+ 15
- 0
models/cloudbrain.go View File

@@ -1073,6 +1073,9 @@ type UserImageConfig struct {
CreateVersion bool `json:"create_version"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
}

type CreateTrainJobParams struct {
@@ -1096,13 +1099,18 @@ type Config struct {
CreateVersion bool `json:"create_version"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
}

type CreateInferenceJobParams struct {
JobName string `json:"job_name"`
Description string `json:"job_desc"`
InfConfig InfConfig `json:"config"`
WorkspaceID string `json:"workspace_id"`
}

type CreateInfUserImageParams struct {
JobName string `json:"job_name"`
Description string `json:"job_desc"`
@@ -1160,6 +1168,9 @@ type TrainJobVersionConfig struct {
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
PreVersionId int64 `json:"pre_version_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
}

type TrainJobVersionUserImageConfig struct {
@@ -1175,6 +1186,9 @@ type TrainJobVersionUserImageConfig struct {
PreVersionId int64 `json:"pre_version_id"`
UserImageUrl string `json:"user_image_url"`
UserCommand string `json:"user_command"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
}

type CreateConfigParams struct {
@@ -1190,6 +1204,7 @@ type CreateConfigParams struct {
LogUrl string `json:"log_url"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
Volumes []Volumes `json:"volumes"`
}

type Parameter struct {


+ 64
- 1
models/cloudbrain_static.go View File

@@ -92,6 +92,17 @@ type HourTimeStatistic struct {
HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"`
HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"`
}
type CloudbrainTypeDuration []struct {
Type int `xorm:"type"`
DurationSum int `xorm:"durationSum"`
CardDurationSum int `xorm:"cardDurationSum"`
Count int `xorm:"count"`
}
type CloudbrainAllDuration struct {
DurationSum int `xorm:"durationSum"`
CardDurationSum int `xorm:"cardDurationSum"`
Count int `xorm:"count"`
}

func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " +
@@ -303,7 +314,7 @@ func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, err
builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}),
)
cond = cond.Or(
builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}),
builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}, builder.Lte{"cloudbrain.start_time": beginTime}),
)
sess.OrderBy("cloudbrain.id ASC")
cloudbrains := make([]*CloudbrainInfo, 0, 10)
@@ -425,3 +436,55 @@ func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime tim
}
return nil
}

func GetCloudbrainTypeCardDuration() (CloudbrainTypeDuration, error) {
query := `
SELECT
cloudbrain.type,
SUM(cloudbrain.duration) as durationSum,
SUM(
COALESCE(cloudbrain.duration *
CASE
WHEN cloudbrain.work_server_number = 0 THEN 1
ELSE COALESCE(cloudbrain.work_server_number, 1)
END *
COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
) as cardDurationSum,
COUNT(*) as count
FROM cloudbrain
LEFT JOIN cloudbrain_spec
ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
GROUP BY cloudbrain.type
`
// 执行查询
var results CloudbrainTypeDuration
if err := x.SQL(query).Find(&results); err != nil {
panic(err)
}
return results, nil
}

func GetCloudbrainAllCardDuration() (CloudbrainAllDuration, error) {
query := `
SELECT
SUM(cloudbrain.duration) as durationSum,
SUM(
COALESCE(cloudbrain.duration *
CASE
WHEN cloudbrain.work_server_number = 0 THEN 1
ELSE COALESCE(cloudbrain.work_server_number, 1)
END *
COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
) as cardDurationSum,
COUNT(*) as count
FROM cloudbrain
LEFT JOIN cloudbrain_spec
ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
`
// 执行查询
var result CloudbrainAllDuration
if _, err := x.SQL(query).Get(&result); err != nil {
panic(err)
}
return result, nil
}

+ 171
- 48
models/user_business_analysis.go View File

@@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)

dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
@@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum
resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount
resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
@@ -583,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
startTime := currentTimeNow.AddDate(0, 0, -1)

CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
IssueCountMap := queryCreateIssue(start_unix, end_unix)

CommentCountMap := queryComment(start_unix, end_unix)
@@ -599,29 +602,25 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
//log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson))
}
//CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix())
CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
CommitDatasetSizeMap, CommitDatasetNumMap, _ := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)
CreateRepoCountMap, _, _ := queryUserCreateRepo(start_unix, end_unix)
LoginCountMap := queryLoginCount(start_unix, end_unix)

OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
CollectImage, CollectedImage := queryImageStars(start_unix, end_unix)
RecommendImage := queryRecommedImage(start_unix, end_unix)

InvitationMap := queryUserInvitationCount(start_unix, end_unix)

DataDate := currentTimeNow.Format("2006-01-02") + " 00:01"
bonusMap := make(map[string]map[string]int)
if isUserYearData(tableName) {
bonusMap = getBonusMap()
log.Info("truncate all data from table:user_summary_current_year ")
statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")
}

cond := "type != 1 and is_active=true"
count, err := sess.Where(cond).Count(new(User))
if err != nil {
@@ -687,6 +686,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap)
dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap)
dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset)
dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset)
dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset)
@@ -719,37 +719,6 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1
}
}
if isUserYearData(tableName) {
//年度数据
subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
mostActiveDay := ""
if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
mostActiveDay = getMostActiveJson(userInfo)
}
scoreMap := make(map[string]float64)
repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
scoreMap["datasetscore"] = datasetscore
codeInfo, codescore := getCodeInfo(dateRecordAll)
scoreMap["codescore"] = codescore
cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
re := &UserSummaryCurrentYear{
ID: dateRecordAll.ID,
Name: dateRecordAll.Name,
Email: dateRecordAll.Email,
Phone: dateRecordAll.Phone,
RegistDate: dateRecordAll.RegistDate,
DateCount: int(subTime.Hours()) / 24,
MostActiveDay: mostActiveDay,
RepoInfo: repoInfo,
DataSetInfo: dataSetInfo,
CodeInfo: codeInfo,
CloudBrainInfo: cloudBrainInfo,
PlayARoll: playARoll,
}
statictisSess.Insert(re)
}
}
if len(dateRecordBatch) > 0 {
err := insertTable(dateRecordBatch, tableName, statictisSess)
@@ -779,6 +748,127 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount))
}

func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) {
sess := x.NewSession()
defer sess.Close()
log.Info("RefreshUserYearTable start....")
statictisSess := xStatistic.NewSession()
defer statictisSess.Close()

log.Info("UserYear StartTime:" + pageStartTime.Format("2006-01-02 15:04:05"))
log.Info("UserYear EndTime time:" + pageEndTime.Format("2006-01-02 15:04:05"))

start_unix := pageStartTime.Unix()
end_unix := pageEndTime.Unix()

CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
IssueCountMap := queryCreateIssue(start_unix, end_unix)

CommentCountMap := queryComment(start_unix, end_unix)

CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime)
if err != nil {
log.Info("query commit code errr.")
} else {
log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap)))
}
CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)

CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)

_, CollectedDataset := queryDatasetStars(start_unix, end_unix)
_, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)

bonusMap := getBonusMap()
log.Info("truncate all data from table:user_summary_current_year ")
statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")

cond := "type != 1 and is_active=true"
count, err := sess.Where(cond).Count(new(User))
if err != nil {
log.Info("query user error. return.")
return
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
userList := make([]*User, 0)
sess.Find(&userList)
for _, userRecord := range userList {
var dateRecordAll UserBusinessAnalysisAll
dateRecordAll.ID = userRecord.ID
dateRecordAll.Email = userRecord.Email
dateRecordAll.Phone = userRecord.PhoneNumber
dateRecordAll.RegistDate = userRecord.CreatedUnix
dateRecordAll.Name = userRecord.Name

dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap)
dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap)
dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap)
dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap)

if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok {
dateRecordAll.CommitCodeSize = 0
} else {
dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines)
}
//dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap)
dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap)
dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap)
dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap)
dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap)

dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap)
dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap)
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)

//年度数据
subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
mostActiveDay := ""
if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
mostActiveDay = getMostActiveJson(userInfo)
}
scoreMap := make(map[string]float64)
repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
scoreMap["datasetscore"] = datasetscore
codeInfo, codescore := getCodeInfo(dateRecordAll)
scoreMap["codescore"] = codescore
cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
re := &UserSummaryCurrentYear{
ID: dateRecordAll.ID,
Name: dateRecordAll.Name,
Email: dateRecordAll.Email,
Phone: dateRecordAll.Phone,
RegistDate: dateRecordAll.RegistDate,
DateCount: int(subTime.Hours()) / 24,
MostActiveDay: mostActiveDay,
RepoInfo: repoInfo,
DataSetInfo: dataSetInfo,
CodeInfo: codeInfo,
CloudBrainInfo: cloudBrainInfo,
PlayARoll: playARoll,
}
statictisSess.Insert(re)
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
log.Info("update user year data finished. ")
}

func isUserYearData(tableName string) bool {
if tableName == "user_business_analysis_current_year" {
currentTimeNow := time.Now()
@@ -998,7 +1088,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static

insertBatchSql := "INSERT INTO public." + tableName +
"(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " +
"VALUES"

for i, record := range dateRecords {
@@ -1007,7 +1097,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static
", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) +
", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) +
", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," +
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")"
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")"
if i < (len(dateRecords) - 1) {
insertBatchSql += ","
}
@@ -1098,6 +1188,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)

CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -1179,7 +1270,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)
dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset)
@@ -1368,6 +1459,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)

result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -1393,6 +1485,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int {
result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset
@@ -1414,6 +1507,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int {
result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset
@@ -1450,6 +1544,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)

result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -1494,10 +1589,6 @@ func getInt(str string) int {
return int(re)
}

func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
}

func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession()
defer sess.Close()
@@ -2278,6 +2369,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int {
return resultMap
}

func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
count, err := sess.Where(cond).Count(new(AiModelConvert))
if err != nil {
log.Info("query AiModelConvert error. return.")
return resultMap
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
aiModelList := make([]*AiModelConvert, 0)
sess.Find(&aiModelList)
log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList)))
for _, aiModelRecord := range aiModelList {
if _, ok := resultMap[aiModelRecord.UserId]; !ok {
resultMap[aiModelRecord.UserId] = 1
} else {
resultMap[aiModelRecord.UserId] += 1
}
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
return resultMap
}

func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) {
sess := x.NewSession()
defer sess.Close()


+ 9
- 0
models/user_business_struct.go View File

@@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLast30Day struct {
@@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLastMonth struct {
@@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisCurrentMonth struct {
@@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisCurrentWeek struct {
@@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisYesterday struct {
@@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysisLastWeek struct {
@@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserAnalysisPara struct {
@@ -616,6 +623,7 @@ type UserBusinessAnalysisAll struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

type UserBusinessAnalysis struct {
@@ -704,4 +712,5 @@ type UserBusinessAnalysis struct {

Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
}

+ 3
- 2
modules/cron/tasks_basic.go View File

@@ -5,10 +5,11 @@
package cron

import (
"code.gitea.io/gitea/modules/setting"
"context"
"time"

"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/modules/urfs_client/urchin"
cloudbrainService "code.gitea.io/gitea/services/cloudbrain"

@@ -296,7 +297,7 @@ func registerHandleCloudbrainDurationStatistic() {
RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{
Enabled: true,
RunAtStart: false,
Schedule: "1 0 * * * ?",
Schedule: "1 1 * * * ?",
}, func(ctx context.Context, _ *models.User, _ Config) error {
repo.CloudbrainDurationStatisticHour()
return nil


+ 12
- 0
modules/modelarts/modelarts.go View File

@@ -283,6 +283,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
Parameter: req.Parameters,
UserImageUrl: req.UserImageUrl,
UserCommand: req.UserCommand,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
},
})
} else {
@@ -303,6 +306,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
Code: req.Spec.SourceSpecId,
},
Parameter: req.Parameters,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
},
})
}
@@ -421,6 +427,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
PreVersionId: req.PreVersionId,
UserImageUrl: req.UserImageUrl,
UserCommand: req.UserCommand,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
},
}, jobId)
} else {
@@ -440,6 +449,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
},
Parameter: req.Parameters,
PreVersionId: req.PreVersionId,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
},
}, jobId)
}


+ 3
- 1
modules/modelarts/resty.go View File

@@ -497,7 +497,7 @@ sendjob:
}

req, _ := json.Marshal(createJobParams)
log.Info("%s", req)
log.Info("postapi json: %s", req)

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
@@ -543,6 +543,8 @@ func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.Create
var result models.CreateTrainJobResult

retry := 0
req, _ := json.Marshal(createJobParams)
log.Info("postapi json: %s", req)

sendjob:
res, err := client.R().


+ 6
- 0
modules/setting/setting.go View File

@@ -585,6 +585,9 @@ var (
TrainJobFLAVORINFOS string
ModelArtsSpecialPools string
ModelArtsMultiNode string
ModelArtsShareAddr string
ModelArtsMountPath string
ModelArtsNasType string
//kanban
IsCloudbrainTimingEnabled bool

@@ -1557,6 +1560,9 @@ func NewContext() {
TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("")
ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("")
ModelArtsShareAddr = sec.Key("ModelArts_Share_Addr").MustString("192.168.0.30:/")
ModelArtsMountPath = sec.Key("ModelArts_Mount_Path").MustString("/cache/sfs")
ModelArtsNasType = sec.Key("ModelArts_Nas_Type").MustString("nfs")

sec = Cfg.Section("elk")
ElkUrl = sec.Key("ELKURL").MustString("")


+ 3
- 3
options/locale/locale_en-US.ini View File

@@ -3125,9 +3125,9 @@ reject_pull_request = `suggested changes for <a href="%s/pulls/%s">%s#%[2]s</a>`
upload_dataset=`upload dataset <a href="%s/datasets">%s</a>`
task_gpudebugjob=`created CPU/GPU type debugging task <a href="%s/cloudbrain/%s">%s</a>`
task_npudebugjob=`created NPU type debugging task <a href="%s/modelarts/notebook/%s">%s</a>`
task_c2entgpudebugjob=`created CPU/GPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2entnpudebugjob=`created NPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2entgcudebugjob=`created GCU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_gpudebugjob=`created CPU/GPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_npudebugjob=`created NPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2ent_gcudebugjob=`created GCU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_nputrainjob=`created NPU training task <a href="%s/modelarts/train-job/%s">%s</a>`
task_inferencejob=`created reasoning task <a href="%s/modelarts/inference-job/%s">%s</a>`
task_benchmark=`created profiling task <a href="%s/cloudbrain/benchmark/%s">%s</a>`


+ 3
- 3
options/locale/locale_zh-CN.ini View File

@@ -3143,9 +3143,9 @@ reject_pull_request=`建议变更 <a href="%s/pulls/%s">%s#%[2]s</a>`
upload_dataset=`上传了数据集文件 <a href="%s/datasets">%s</a>`
task_gpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/cloudbrain/%s">%s</a>`
task_npudebugjob=`创建了NPU类型调试任务 <a href="%s/modelarts/notebook/%s">%s</a>`
task_c2entgpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2entnpudebugjob=`创建了NPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2entgcudebugjob=`创建了GCU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_gpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_npudebugjob=`创建了NPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2ent_gcudebugjob=`创建了GCU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_nputrainjob=`创建了NPU类型训练任务 <a href="%s/modelarts/train-job/%s">%s</a>`
task_inferencejob=`创建了推理任务 <a href="%s/modelarts/inference-job/%s">%s</a>`
task_benchmark=`创建了评测任务 <a href="%s/cloudbrain/benchmark/%s">%s</a>`


+ 3
- 2
public/home/home.js View File

@@ -244,11 +244,11 @@ document.onreadystatechange = function () {
html += " <a href=\"" + getRepoLink(record) + "\" rel=\"nofollow\">" + getRepotext(record) + "</a>"
}
else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30"
|| record.OpType == "31" || record.OpType == "32" || record.OpType == "33" || record.OpType == "39" || record.OpType == "40" || record.OpType == "41"){
|| record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){
html += recordPrefix + actionName;
html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
}
else if(record.OpType == "25" || record.OpType == "29"){
else if(record.OpType == "25" || record.OpType == "29" || record.OpType == "39" || record.OpType == "40" || record.OpType == "41"){
html += recordPrefix + actionName;
html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
}
@@ -298,6 +298,7 @@ function getTaskLink(record){
}else if(record.OpType == 39 || record.OpType == 40 || record.OpType == 41){
re = re + "/grampus/notebook/" + record.Content;
}
re = encodeURI(re);
return re;
}


+ 55
- 8
routers/api/v1/repo/cloudbrain.go View File

@@ -9,6 +9,7 @@ import (
"bufio"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path"
@@ -647,6 +648,19 @@ func CloudbrainDownloadLogFile(ctx *context.Context) {
}
}

existStr := ""
if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
if job.Type == models.TypeCloudBrainOne {
result, err := cloudbrain.GetJob(job.JobID)
if err == nil && result != nil {
jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
taskRoles := jobRes.TaskRoles
taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
existStr = taskRes.TaskStatuses[0].ExitDiagnostics
}
}
}

logDir := "/model"
if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) {
logDir = cloudbrain.ResultPath
@@ -664,17 +678,30 @@ func CloudbrainDownloadLogFile(ctx *context.Context) {
}
}
if fileName != "" {
prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir
url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName)
prefix := "/" + setting.CBCodePathPrefix + job.JobName + "/model"
filePath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + prefix + "/" + fileName
// Read the file contents into a byte slice
data, err := ioutil.ReadFile(filePath)
if err != nil {
log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("ReadFile", err)
return
}

// Set the appropriate response headers
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)

// Write the file contents to the response
if _, err := ctx.Resp.Write(data); err != nil {
ctx.ServerError("Write", err)
return
}
if _, err := ctx.Resp.Write([]byte(existStr)); err != nil {
log.Error("Write failed: %v", err.Error(), ctx.Data["msgID"])
return
}
log.Info("fileName=" + fileName)
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
} else {
log.Info("fileName is null.")

}
}

@@ -760,8 +787,28 @@ func CloudbrainGetLog(ctx *context.APIContext) {
content = result["Content"].(string)
}

if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
content = content + ctx.Data["existStr"].(string)
if (job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference)) && job.Type == models.TypeCloudBrainOne && job.Status == string(models.JobFailed) {
if ctx.Data["existStr"] != nil {
if baseLine == "" && order == "desc" && result["Lines"].(int) == 0 {
result["Lines"] = 1
result["EndLine"] = 1
content = content + ctx.Data["existStr"].(string)
}

if result["Lines"].(int) == 0 && result["StartLine"] == result["EndLine"] && result["StartLine"].(int) != 0 {
content = content + ctx.Data["existStr"].(string)
result["Lines"] = 1
result["StartLine"] = result["StartLine"].(int) - 1
}
if result["Lines"].(int) == 1 && result["StartLine"] == result["EndLine"] {
result["Lines"] = 0
result["StartLine"] = result["StartLine"].(int) + 1
}
}
} else {
if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
content = content + ctx.Data["existStr"].(string)
}
}

logFileName := result["FileName"]


+ 36
- 60
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -103,86 +103,62 @@ func GetAllCloudbrainsOverview(ctx *context.Context) {
})
}
func GetOverviewDuration(ctx *context.Context) {
recordCloudbrain, err := models.GetRecordBeginTime()
if err != nil {
log.Error("Can not get recordCloudbrain", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
return
}
recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix
now := time.Now()
endTime := now
var workServerNumber int64
var cardNum int64
durationSum := 0
cardDurationSum := 0

durationAllSum := int64(0)
cardDuSum := int64(0)
cloudBrainOneCardDuSum := 0
cloudBrainTwoCardDuSum := 0
c2NetCardDuSum := 0
cDNetCardDuSum := 0

cloudBrainOneCardDuSum := int64(0)
cloudBrainTwoCardDuSum := int64(0)
c2NetCardDuSum := int64(0)
cDNetCardDuSum := int64(0)
cloudBrainOneDuration := 0
cloudBrainTwoDuration := 0
c2NetDuration := 0
cDCenterDuration := 0

cloudBrainOneDuration := int64(0)
cloudBrainTwoDuration := int64(0)
c2NetDuration := int64(0)
cDCenterDuration := int64(0)

cloudbrains, _, err := models.CloudbrainAllKanBan(&models.CloudbrainsOptions{
Type: models.TypeCloudBrainAll,
BeginTimeUnix: int64(recordBeginTime),
EndTimeUnix: endTime.Unix(),
})
cloudbrainTypeDuration, err := models.GetCloudbrainTypeCardDuration()
if err != nil {
ctx.ServerError("Get cloudbrains failed:", err)
log.Error("GetCloudbrainTypeCardDuration err!", err)
return
}
models.LoadSpecs4CloudbrainInfo(cloudbrains)

for _, cloudbrain := range cloudbrains {
cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.Spec != nil {
cardNum = int64(cloudbrain.Cloudbrain.Spec.AccCardsNum)
} else {
cardNum = 1
for _, result := range cloudbrainTypeDuration {
if result.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration = result.DurationSum
cloudBrainOneCardDuSum = result.CardDurationSum
}
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
workServerNumber = int64(cloudbrain.Cloudbrain.WorkServerNumber)
} else {
workServerNumber = 1
if result.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration = result.DurationSum
cloudBrainTwoCardDuSum = result.CardDurationSum
}
duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration)
CardDuration := workServerNumber * int64(cardNum) * duration

if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration += duration
cloudBrainOneCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration += duration
cloudBrainTwoCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
c2NetDuration += duration
c2NetCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
cDCenterDuration += duration
cDNetCardDuSum += CardDuration
if result.Type == models.TypeC2Net {
c2NetDuration = result.DurationSum
c2NetCardDuSum = result.CardDurationSum
}

durationAllSum += duration
cardDuSum += CardDuration
if result.Type == models.TypeCDCenter {
cDCenterDuration = result.DurationSum
cDNetCardDuSum = result.CardDurationSum
}
}
cloudbrainAllDuration, err := models.GetCloudbrainAllCardDuration()
if err != nil {
log.Error("GetCloudbrainAllCardDuration err!", err)
return
}
durationSum = cloudbrainAllDuration.DurationSum
cardDurationSum = cloudbrainAllDuration.CardDurationSum

ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudBrainOneCardDuSum": cloudBrainOneCardDuSum,
"cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum,
"c2NetCardDuSum": c2NetCardDuSum,
"cDNetCardDuSum": cDNetCardDuSum,
"cardDuSum": cardDuSum,
"cardDuSum": cardDurationSum,

"cloudBrainOneDuration": cloudBrainOneDuration,
"cloudBrainTwoDuration": cloudBrainTwoDuration,
"c2NetDuration": c2NetDuration,
"cDCenterDuration": cDCenterDuration,
"durationSum": durationAllSum,
"durationSum": durationSum,
})
}



+ 1
- 1
routers/repo/ai_model_convert.go View File

@@ -603,7 +603,7 @@ func stopModelConvert(id string) error {
}

func StopModelConvertApi(ctx *context.Context) {
id := ctx.Params(":id")
id := ctx.Query("id")
log.Info("stop model convert start.id=" + id)
err := stopModelConvert(id)
if err == nil {


+ 4
- 1
routers/repo/aisafety.go View File

@@ -847,6 +847,9 @@ func createForGPU(ctx *context.Context, jobName string) error {
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
os.RemoveAll(codePath)

gitRepo, _ := git.OpenRepository(repo.RepoPath())
commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName)

if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
return errors.New("system error")
@@ -891,7 +894,7 @@ func createForGPU(ctx *context.Context, jobName string) error {
BranchName: cloudbrain.DefaultBranchName,
BootFile: BootFile,
Params: Params,
CommitID: "",
CommitID: commitID,
ModelName: modelName,
ModelVersion: modelVersion,
CkptName: CkptName,


+ 74
- 120
routers/repo/cloudbrain_statistic.go View File

@@ -14,13 +14,7 @@ import (
)

func CloudbrainDurationStatisticHour() {
defer func() {
err := recover()
if err == nil {
return
}
}()
if setting.IsCloudbrainTimingEnabled {
if setting.IsCloudbrainTimingEnabled {
var statisticTime time.Time
var count int64
recordDurationUpdateTime, err := models.GetDurationRecordUpdateTime()
@@ -35,17 +29,16 @@ func CloudbrainDurationStatisticHour() {
statisticTime = currentTime
}

err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Add(-1*time.Hour).Unix()), timeutil.TimeStamp(currentTime.Unix()))
err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Unix()), timeutil.TimeStamp(currentTime.Unix()))
if err != nil {
log.Error("DeleteCloudbrainDurationStatistic failed", err)
}
statisticTime = statisticTime.Add(+1 * time.Hour)
for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) {
countEach := summaryDurationStat(statisticTime)
count += countEach
statisticTime = statisticTime.Add(+1 * time.Hour)
}
log.Info("summaryDurationStat count: %v", count)
}
}
func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 {
@@ -71,15 +64,18 @@ func summaryDurationStat(statisticTime time.Time) int64 {

ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime)
if err != nil {
log.Info("GetCloudbrainByTime err: %v", err)
log.Error("GetCloudbrainByTime err: %v", err)
return 0
}
models.LoadSpecs4CloudbrainInfo(ciTasks)
cloudBrainCenterCodeAndCardTypeInfo, cloudbrainMap := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime)
err = models.LoadSpecs4CloudbrainInfo(ciTasks)
if err != nil {
log.Error("LoadSpecs4CloudbrainInfo err: %v", err)
}
cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, int(beginTime), int(endTime))

resourceQueues, err := models.GetCanUseCardInfo()
if err != nil {
log.Info("GetCanUseCardInfo err: %v", err)
log.Error("GetCanUseCardInfo err: %v", err)
return 0
}

@@ -91,56 +87,45 @@ func summaryDurationStat(statisticTime time.Time) int64 {
cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] += resourceQueue.CardsTotalNum * 1 * 60 * 60
}
}

for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo {
for cardType, cardDuration := range CardTypes {
cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType]
if cloudbrainTable != nil {
if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok {
cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0
}
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: cloudbrainTable.Cluster,
AiCenterName: GetAiCenterNameByCode(centerCode, "zh-CN"),
AiCenterCode: centerCode,
AccCardType: cardType,
CardsUseDuration: cardDuration,
CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType],
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType)
}
}
}

for key, cardsTotalDuration := range cardsTotalDurationMap {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: 0,
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
if _, ok := cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]]; ok {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]],
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
} else {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: 0,
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
}
count++
}

log.Info("finish summary cloudbrainDurationStat")
return count
}

@@ -159,33 +144,21 @@ func GetAiCenterNameByCode(centerCode string, language string) string {
return aiCenterName
}

func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) (map[string]map[string]int, map[string]*models.Cloudbrain) {
func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, hourBeginTime int, hourEndTime int) map[string]map[string]int {
var WorkServerNumber int
var AccCardsNum int
cloudbrainMap := make(map[string]*models.Cloudbrain)
cloudBrainCenterCodeAndCardType := make(map[string]map[string]int)
for _, cloudbrain := range ciTasks {
if cloudbrain.Cloudbrain.StartTime == 0 {
cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
}
if cloudbrain.Cloudbrain.EndTime == 0 {
cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix())
}
cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.Spec != nil {
if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if cloudbrain.Cloudbrain.Spec != nil {
cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain
}
}
}

cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.StartTime == 0 {
cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
}
if cloudbrain.Cloudbrain.EndTime == 0 {
cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix
if cloudbrain.Cloudbrain.Status == string(models.JobRunning) {
cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix())
} else {
cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.StartTime + timeutil.TimeStamp(cloudbrain.Cloudbrain.Duration)
}
}
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber
@@ -197,55 +170,36 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be
} else {
AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
}
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter]; !ok {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter] = make(map[string]int)
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter]; !ok {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter] = make(map[string]int)
}
taskStartTime := int(cloudbrain.Cloudbrain.StartTime)
taskEndTime := int(cloudbrain.Cloudbrain.EndTime)
if cloudbrain.Cloudbrain.Spec != nil {
if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) && cloudbrain.Cloudbrain.DeletedAt.IsZero() {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0
}
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime)
} else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime)
}
} else {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime)
} else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime)
}
}
}
}

return cloudBrainCenterCodeAndCardType, cloudbrainMap
return cloudBrainCenterCodeAndCardType
}

func CloudbrainUpdateHistoryData(ctx *context.Context) {


+ 2
- 2
routers/repo/grampus.go View File

@@ -974,7 +974,7 @@ func GrampusStopJob(ctx *context.Context) {
if task.Status == models.GrampusStatusStopped || task.Status == models.GrampusStatusFailed || task.Status == models.GrampusStatusSucceeded {
log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
resultCode = "-1"
errorMsg = "System error"
errorMsg = ctx.Tr("cloudbrain.Already_stopped")
break
}

@@ -982,7 +982,7 @@ func GrampusStopJob(ctx *context.Context) {
if err != nil {
log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
resultCode = strconv.Itoa(res.ErrorCode)
errorMsg = res.ErrorMsg
errorMsg = ctx.Tr("cloudbrain.Stopped_failed")
break
}
oldStatus := task.Status


+ 4
- 1
routers/repo/modelarts.go View File

@@ -346,9 +346,12 @@ func NotebookShow(ctx *context.Context) {
}

func GetModelDownload(task *models.Cloudbrain) models.ModelDownload {
index := strings.Index(task.PreTrainModelUrl, "/")
key := task.PreTrainModelUrl[index+1:] + task.CkptName
url, _ := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key)
modelDownload := models.ModelDownload{
Name: task.CkptName,
DownloadLink: "s3://" + task.PreTrainModelUrl + task.CkptName,
DownloadLink: url,
IsDelete: false,
}



+ 12
- 0
routers/repo/user_data_analysis.go View File

@@ -21,6 +21,7 @@ import (
const (
PAGE_SIZE = 2000
Excel_File_Path = "/useranalysis/"
USER_YEAR = 2022
)

func getUserMetricsExcelHeader(ctx *context.Context) map[string]string {
@@ -104,6 +105,7 @@ func getExcelHeader(ctx *context.Context) map[string]string {
excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount"))
excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount"))

excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset"))
@@ -178,6 +180,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)
@@ -256,6 +260,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)
@@ -714,6 +720,12 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) {
log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05"))
log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05"))
warnEmailMessage := "用户统计信息入库失败,请尽快定位。"

startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location())
endYear := startYear.AddDate(1, 0, 0)

models.RefreshUserYearTable(startYear, endYear)

//query wiki data
log.Info("start to time count data")
wikiMap, err := queryWikiCountMap(startTime, endTime)


+ 1
- 1
services/cloudbrain/cloudbrainTask/count.go View File

@@ -16,7 +16,7 @@ type StatusInfo struct {

var CloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)}
var CloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)}
var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning, models.GrampusStatusStopping}
var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning}
var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainOne): {
CloudBrainTypes: []int{models.TypeCloudBrainOne},
JobType: []models.JobType{models.JobTypeDebug},


+ 1
- 1
services/socketwrap/clientManager.go View File

@@ -10,7 +10,7 @@ import (
"github.com/elliotchance/orderedmap"
)

var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 41}
var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 39, 40, 41}

type ClientsManager struct {
Clients *orderedmap.OrderedMap


+ 18
- 25
templates/admin/cloudbrain/list.tmpl View File

@@ -96,16 +96,9 @@
{{end}}
<!-- {{$JobID}} -->
<div class="two wide column nowrap" style="width:10% !important;">
{{if and (eq .JobType "DEBUG") (eq .Cloudbrain.Type 0)}}
{{if eq .JobType "DEBUG"}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain/{{$JobID}}{{else}}/modelarts/notebook/{{$JobID}}{{end}}"
title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
</a>
{{else if eq .JobType "DEBUG"}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}/grampus/notebook/{{$JobID}}"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}"
title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -142,13 +135,13 @@
</div>
<!-- 集群 -->
<div class="one wide column text center nowrap" style="width:6% !important;">
<span style="font-size: 12px;" class="cluster_{{.DisplayJobName}}_{{$JobID}}">{{if .Cluster}}{{.Cluster}}{{else}}--{{end}}</span>
<span style="font-size: 12px;" class="cluster_{{.DisplayJobName}}_{{$JobID}}">{{if .Cluster}}{{.Cluster}}{{else}}--{{end}}</span>
</div>
<!-- 任务状态 -->
<div class="two wide column text center nowrap"
style="width: 6% !important;">
<span class="job-status" id="{{$JobID}}"
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{else if eq .JobType "INFERENCE"}}/modelarts/inference-job{{else if eq .JobType "TRAIN"}}/modelarts/train-job{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "INFERENCE"}}/modelarts/inference-job{{else if eq .JobType "TRAIN"}}/modelarts/train-job{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
<span><i id="{{$JobID}}-icon" style="vertical-align: middle;"
class="{{.Status}}"></i><span id="{{$JobID}}-text"
@@ -177,7 +170,7 @@
</div>
<!-- 智算中心 -->
<div class="one wide column text center nowrap" style="width:8% !important;">
<span style="font-size: 12px;" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
<span style="font-size: 12px;" id="cluster-{{$JobID}}" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
</div>
<!-- XPU类型 -->
<div class="one wide column text center nowrap" style="width:8% !important;">
@@ -240,7 +233,7 @@
{{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}}
<a style="margin: 0 1rem;" id="ai-debug-{{$JobID}}"
class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}}disabled {{else}}blue {{end}}button'
data-jobid="{{$JobID}}"
data-jobid="{{$JobID}}"
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/'>
{{$.i18n.Tr "repo.debug"}}
</a>
@@ -276,7 +269,7 @@
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
class='ui basic ai_stop {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED" "STOPPING"}}disabled {{else}} blue {{end}}button'
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else if eq .JobType "BENCHMARK" }}/cloudbrain/benchmark{{else if eq .ComputeResource "NPU" }}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/stop'
data-jobid="{{$JobID}}">
data-jobid="{{$JobID}}" data-bootfile="{{.BootFile}}">
{{$.i18n.Tr "repo.stop"}}
</a>
</form>
@@ -311,17 +304,17 @@
</a>
</form>
{{else}}
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{ if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{ if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?isadminpage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?isadminpage=true"
data-version="" class="ui basic ai_delete blue button"
style="border-radius: .28571429rem;">
{{$.i18n.Tr "repo.delete"}}
</a>
</form>
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?ishomepage=true"
data-version="{{.VersionName}}" class="ui basic ai_delete blue button"
style="border-radius: .28571429rem;">
{{$.i18n.Tr "repo.delete"}}
</a>
</form>
{{end}}
</div>
</div>


+ 3
- 77
templates/repo/cloudbrain/new.tmpl View File

@@ -125,7 +125,7 @@
<input id="store_category" type="hidden" name="get_benchmark_category">
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2 {{if not .Branches}}error{{end}}" id="code_version"
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
@@ -216,7 +216,7 @@
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel"
@@ -231,6 +231,7 @@
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
let form = document.getElementById('form_id');
$('#messageInfo').css('display', 'none')
@@ -239,67 +240,6 @@
context.value = ''
$(".icon.icons").css("visibility", "hidden")
}
var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function (e) {
if (!isValidate) return false;
if(createFlag) return false
let value_task = $("input[name='display_job_name']").val()
let value_image = $("input[name='image']").val()
let value_data = $("input[name='attachment']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if (!flag) {
$('#messageInfo').css('display', 'block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
if($('input[name="model_name"]').val() && !$('input[name="ckpt_name"]').val()){
$('input[name="ckpt_name"]').parent().addClass("error")
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value", min_value_task)
createFlag = true
document.getElementById("mask").style.display = "block"
}

// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function () {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}

}

$('#cloudbrain_benchmark_category')
.dropdown({
placeholder: "选择数据集类别",
@@ -334,20 +274,6 @@
}
})
})

$('.ui.green.button').click(function () {
if (!$('input[name="isBranches"]').val()) {
return false
}
selected_value = $("#cloudbrain_benchmark_category").val()
$('#store_category').attr("value", selected_value)
validate();
})

$(".question.circle.icon").hover(function () {
$(this).popup("show");
});

;(function() {
var SPECS = {{ .debug_specs }};
var showPoint = {{ .CloudBrainPaySwitch }};


+ 4
- 66
templates/repo/grampus/notebook/gpu/new.tmpl View File

@@ -91,7 +91,7 @@
<div class="ui divider"></div>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2 {{if not .Branches}}error{{end}}" id="code_version"
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
@@ -140,7 +140,7 @@
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel"
@@ -155,70 +155,8 @@
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
let form = document.getElementById('form_id');
var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function (e) {
if (!isValidate) return false;
if(createFlag) return false
let value_task = $("input[name='display_job_name']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if (!flag) {
$('#messageInfo').css('display', 'block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
if($('input[name="model_name"]').val() && !$('input[name="ckpt_name"]').val()){
$('input[name="ckpt_name"]').parent().addClass("error")
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value", min_value_task)
createFlag = true
document.getElementById("mask").style.display = "block"
}
// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function () {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}

}

$(".question.circle.icon").hover(function () {
$(this).popup("show");
});

<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
;(function() {
var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }};


+ 4
- 81
templates/repo/grampus/notebook/npu/new.tmpl View File

@@ -76,7 +76,7 @@
<div class="ui divider"></div>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2 {{if not .Branches}}error{{end}}" id="code_version"
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
@@ -128,7 +128,7 @@
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel" href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
@@ -141,85 +141,8 @@
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
// 判断必填选项是否填写正确
let form = document.getElementById('form_id');
$('#messageInfo').css('display','none')
var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function(e){
if(!isValidate) return false;
if(createFlag) return false;
let value_task = $("input[name='display_job_name']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if(!flag){
$('#messageInfo').css('display','block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
if($('input[name="model_name"]').val() && !$('input[name="ckpt_name"]').val()){
$('input[name="ckpt_name"]').parent().addClass("error")
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value",min_value_task)
document.getElementById("mask").style.display = "block"
createFlag = true
}
// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}
$(document).ready(function(){
$('input[name="image"]').val($('.cloudbrain_image').dropdown('get text'))
$('.cloudbrain_image.ui.search.dropdown')
.dropdown({
onChange: function(value, text, $selectedItem) {
$('input[name="image"]').val(text)
}
})
$(document).keydown(function(event){
if(event.keyCode==13){
event.preventDefault();
}
});
});
$(".question.circle.icon").hover(function () {
$(this).popup("show");
});
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
;(function() {
var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }};


+ 1
- 11
templates/repo/grampus/notebook/show.tmpl View File

@@ -298,17 +298,7 @@
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.output_storage_path"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="model_storage_path">
{{$.model_path}}
</div>
</td>
</tr>
{{end}}
{{if eq .ComputeResource "NPU"}}


+ 2
- 2
templates/repo/grampus/trainjob/gpu/new.tmpl View File

@@ -90,14 +90,14 @@
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<a class="active item" href="{{.RepoLink}}/grampus/train-job/gpu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU
</a>
<a {{if.NPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<a class="item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>


+ 2
- 2
templates/repo/grampus/trainjob/npu/new.tmpl View File

@@ -76,14 +76,14 @@
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<a class="item" href="{{.RepoLink}}/grampus/train-job/gpu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU
</a>
<a {{if.NPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<a class="active item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>


+ 2
- 68
templates/repo/modelarts/notebook/new.tmpl View File

@@ -105,7 +105,7 @@
</div>
<div class="inline field">
<label class="label-fix-width"></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel" href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
@@ -118,74 +118,8 @@
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
// 判断必填选项是否填写正确
let form = document.getElementById('form_id');
$('#messageInfo').css('display','none')
var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function(e){
if(!isValidate) return false;
if(createFlag) return false;
let value_task = $("input[name='display_job_name']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if(!flag){
$('#messageInfo').css('display','block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
if($('input[name="model_name"]').val() && !$('input[name="ckpt_name"]').val()){
$('input[name="ckpt_name"]').parent().addClass("error")
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value",min_value_task)
document.getElementById("mask").style.display = "block"
createFlag = true
}
// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}
$(document).ready(function(){
$(document).keydown(function(event){
if(event.keyCode==13){
event.preventDefault();
}
});
});

;(function() {
var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }};


+ 8
- 16
templates/user/dashboard/cloudbrains.tmpl View File

@@ -78,16 +78,9 @@
{{end}}
<!-- {{$JobID}} -->
<div class="three wide column nowrap" style="width:12% !important">
{{if and (eq .JobType "DEBUG") (eq .Cloudbrain.Type 0)}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain/{{$JobID}}{{else}}/modelarts/notebook/{{$JobID}}{{end}}"
title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
</a>
{{else if eq .JobType "DEBUG"}}
{{if eq .JobType "DEBUG"}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}/grampus/notebook/{{$JobID}}"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}"
title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -130,7 +123,7 @@
<div class="two wide column text center nowrap"
style="width: 8% !important;">
<span class="job-status" id="{{$JobID}}"
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts{{end}}/inference-job{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts{{end}}/inference-job{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-jobid="{{$JobID}}" data-version="{{.VersionName}}"
data-bootfile="{{.BootFile}}">
<span><i id="{{$JobID}}-icon" style="vertical-align: middle;"
@@ -162,7 +155,7 @@
<!-- 智算中心 -->
<div class="one wide column text center nowrap" style="width:8% !important;">
<span style="font-size: 12px;" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
<span style="font-size: 12px;" id="cluster-{{$JobID}}" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
</div>
<!-- XPU类型 -->
<div class="one wide column text center nowrap" style="width:10% !important;">
@@ -234,7 +227,7 @@
{{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}}
<form id="stopForm-{{$JobID}}" style="margin-left:-1px;">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
class='ui basic ai_stop {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED" "STOPPING" "CREATE_FAILED"}}disabled {{else}} blue {{end}}button'
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else if eq .JobType "BENCHMARK" }}/cloudbrain/benchmark{{else if eq .ComputeResource "NPU" }}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/stop'
data-jobid="{{$JobID}}" data-bootfile="{{.BootFile}}">
@@ -281,18 +274,17 @@
</form>
{{else}}
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?ishomepage=true"
data-version="" class="ui basic ai_delete blue button"
data-version="{{.VersionName}}" class="ui basic ai_delete blue button"
style="border-radius: .28571429rem;">
{{$.i18n.Tr "repo.delete"}}
</a>
</form>
{{end}}
{{end}}
</div>
</div>
</div>


+ 7
- 8
templates/user/dashboard/feeds.tmpl View File

@@ -1,4 +1,3 @@
<script>var Feeds = {{.Feeds}}</script>
{{range .Feeds}}
<div class="news">
<div class="ui left">
@@ -76,13 +75,7 @@
{{else if eq .GetOpType 25}}
{{$.i18n.Tr "action.task_gpudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 26}}
{{$.i18n.Tr "action.task_npudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 39}}
{{$.i18n.Tr "action.task_c2entgpudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 40}}
{{$.i18n.Tr "action.task_c2entnpudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 41}}
{{$.i18n.Tr "action.task_c2entgcudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{$.i18n.Tr "action.task_npudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 27}}
{{$.i18n.Tr "action.task_nputrainjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 28}}
@@ -99,6 +92,12 @@
{{$.i18n.Tr "action.task_c2netgputrainjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 35}}
{{$.i18n.Tr "action.dataset_recommended" .GetRepoLink (index .GetIssueInfos 1 | RenderEmoji) | Str2html}}
{{else if eq .GetOpType 39}}
{{$.i18n.Tr "action.task_c2net_npudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 40}}
{{$.i18n.Tr "action.task_c2net_gpudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 41}}
{{$.i18n.Tr "action.task_c2ent_gcudebugjob" .GetRepoLink .Content .RefName | Str2html}}
{{end}}
</p>
{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}


+ 4
- 1
web_src/js/features/cloudbrainShow.js View File

@@ -862,11 +862,14 @@ export default async function initCloudrainSow() {
!element.IsDir &&
loadCheckpointFile.includes(ckptSuffix[ckptSuffix.length - 1])
) {
html += `<div class="item" data-value=${element.FileName}>${element.FileName}</div>`;
html += `<div class="item" data-value="${element.FileName}">${element.FileName}</div>`;
}
});
$("#model_checkpoint").append(html);
$("#select_model_checkpoint").removeClass("loading");
if (html) {
$("#select_model_checkpoint").removeClass("error");
}
const initVersionText = $(
"#model_checkpoint div.item:first-child"
).text();


+ 4
- 1
web_src/js/features/cloudrbanin.js View File

@@ -56,7 +56,10 @@ export default async function initCloudrain() {
const ID = data.ID || data.JobID;
const status = data.JobStatus;
const duration = data.JobDuration;
$("#duration-" + ID).text(duration);
const aiCenter = data.AiCenter || '--'
$("#duration-" + ID).text(duration);
$("#cluster-" + ID).text(aiCenter);
$("#" + versionname + "-ai_center").text(data.AiCenter);
if (status != status_text) {
$("#" + ID + "-icon")
.removeClass()


+ 12
- 0
web_src/js/standalone/cloudbrainNew.js View File

@@ -108,9 +108,14 @@
identifier: "spec_id",
rules: [{ type: "empty" }],
},
branch_name: {
identifier: "branch_name",
rules: [{ type: "empty" }],
},
},
onSuccess: function () {
// $('.ui.page.dimmer').dimmer('show')
document.getElementById("mask").style.display = "block";
isValidate = true;
},
@@ -159,6 +164,9 @@
let name2 = $("#flaver_name .text").text();
$("input#ai_engine_name").val(name1);
$("input#ai_flaver_name").val(name2);
if ($(".cloudbrain_image .text").text()) {
$("input[name='image']").val($(".cloudbrain_image .text").text());
}
}

validate();
@@ -168,6 +176,10 @@
if (!paramNotValue) {
return false;
}
if($('input[name="model_name"]').val() && !$('input[name="ckpt_name"]').val()){
$('input[name="ckpt_name"]').parent().addClass("error")
return false
}
validate();
});
})();

+ 5
- 1
web_src/vuepages/pages/notebook/debug/index.vue View File

@@ -344,7 +344,7 @@ export default {
location.href=`${AppSubUrl}/authentication/wechat/bind`
}
if(err.response.status===401){
location.href=`${AppSubUrl}/user/login`
location.href=`${AppSubUrl}/user/login?redirect_to=${location.origin}${location.pathname}?type=login`
return
}
this.btnStatus[index]=0
@@ -411,6 +411,10 @@ export default {
this.fileInfo.project_name = selfData.getAttribute('data-project')
this.fileInfo.sign_name = selfData.getAttribute('data-name')
let that = this;
if(new URLSearchParams(window.location.search).get("type")==='login'){
that.getNotebookInfo()
that.dialogVisible = true;
}
document
.querySelector("#notebook-debug")
.addEventListener("click", function () {


Loading…
Cancel
Save