From 67730854bdbce954472461caf6bee6a864e0f72c Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 9 Jan 2023 10:18:20 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E6=89=93=E6=A6=9C=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_analysis_for_activity.go | 14 ++++++++++ models/user_business_analysis.go | 52 ++++++++++++++++++++++++++++++++++++ models/user_business_struct.go | 6 ++--- 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index 99ff990ce..39f8c545d 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "fmt" "time" @@ -450,15 +451,28 @@ func QueryUserLoginInfo(userIds []int64) []*UserLoginLog { return loginList } +var WeekBonusData = make(map[int64][]int) + func QueryUserAnnualReport(userId int64) *UserSummaryCurrentYear { statictisSess := xStatistic.NewSession() defer statictisSess.Close() log.Info("userId=" + fmt.Sprint(userId)) + if len(WeekBonusData) == 0 { + WeekBonusData = getBonusWeekDataMap() + } reList := make([]*UserSummaryCurrentYear, 0) err := statictisSess.Select("*").Table(new(UserSummaryCurrentYear)).Where("id=" + fmt.Sprint(userId)).Find(&reList) if err == nil { if len(reList) > 0 { + record, ok := WeekBonusData[userId] + if ok { + bonusInfo := make(map[string]int) + bonusInfo["order"] = record[0] + bonusInfo["money"] = record[1] + bonusInfoJson, _ := json.Marshal(bonusInfo) + reList[0].WeekBonusData = string(bonusInfoJson) + } return reList[0] } } else { diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index d5ab871ce..9f81f1c27 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -880,6 +880,50 @@ func isUserYearData(tableName string) bool { return false } +func getBonusWeekDataMap() map[int64][]int { + bonusMap := make(map[int64][]int) + url := setting.RecommentRepoAddr + "bonus/weekdata/record.txt" + content, err := GetContentFromPromote(url) + if err == nil { + filenames := strings.Split(content, "\n") + for i := 0; i < len(filenames); i++ { + url = setting.RecommentRepoAddr + "bonus/weekdata/" + filenames[i] + csvContent, err1 := GetContentFromPromote(url) + if err1 == nil { + //read csv + lines := strings.Split(csvContent, "\n") + for j := 1; j < len(lines); j++ { + aLine := strings.Split(lines[j], ",") + if len(aLine) < 4 { + continue + } + userId := getInt64Value(aLine[0]) + order := getIntValue(aLine[2]) + money := getIntValue(aLine[3]) + //email := lines[2] + record, ok := bonusMap[userId] + if !ok { + record = make([]int, 2) + record[0] = order + record[1] = money + bonusMap[userId] = record + } else { + if record[0] > order { + record[0] = order + record[1] = money + } else { + if record[0] == order && record[1] < money { + record[1] = money + } + } + } + } + } + } + } + return bonusMap +} + func getBonusMap() map[string]map[string]int { bonusMap := make(map[string]map[string]int) url := setting.RecommentRepoAddr + "bonus/record.txt" @@ -923,6 +967,14 @@ func getIntValue(val string) int { return 0 } +func getInt64Value(val string) int64 { + i, err := strconv.ParseInt(val, 10, 64) + if err == nil { + return i + } + return 0 +} + func getPlayARoll(bonusMap map[string]map[string]int, userName string, scoreMap map[string]float64) string { bonusInfo := make(map[string]string) record, ok := bonusMap[userName] diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 00c7f6176..61f499732 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -18,9 +18,9 @@ type UserSummaryCurrentYear struct { CodeInfo string `xorm:"varchar(500)"` //代码提交次数，提交总代码行数，最晚的提交时间 CloudBrainInfo string `xorm:"varchar(1000)"` //，创建了XX 个云脑任务，调试任务XX 个，训练任务XX 个，推理任务XX 个，累计运行了XXXX 卡时，累计节省xxxxx 元 //这些免费的算力资源分别有，XX% 来自鹏城云脑1，XX% 来自鹏城云脑2，XX% 来自智算网络 - PlayARoll string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励 - - Label string `xorm:"varchar(500)"` + PlayARoll string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励 + WeekBonusData string `xorm:"-"` + Label string `xorm:"varchar(500)"` } type UserBusinessAnalysisCurrentYear struct { From a3d4a917d05836955a027aa5554dcd9872904174 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 9 Jan 2023 10:30:37 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=B9=B4=E5=BA=A6=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=AD=E5=A2=9E=E5=8A=A0=E4=B8=AA=E4=BA=BA=E5=90=8D=E6=AC=A1?= =?UTF-8?q?=E6=9C=80=E5=A5=BD=E7=9A=84=E5=91=A8=E5=BA=A6=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_analysis_for_activity.go | 2 ++ models/user_business_analysis.go | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index 39f8c545d..6d8201e69 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -470,6 +470,8 @@ func QueryUserAnnualReport(userId int64) *UserSummaryCurrentYear { bonusInfo := make(map[string]int) bonusInfo["order"] = record[0] bonusInfo["money"] = record[1] + bonusInfo["week"] = record[2] + bonusInfo["num"] = record[3] bonusInfoJson, _ := json.Marshal(bonusInfo) reList[0].WeekBonusData = string(bonusInfoJson) } diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 9f81f1c27..c18e1f51a 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -900,20 +900,27 @@ func getBonusWeekDataMap() map[int64][]int { userId := getInt64Value(aLine[0]) order := getIntValue(aLine[2]) money := getIntValue(aLine[3]) + week, num := getWeekAndNum(filenames[i]) //email := lines[2] record, ok := bonusMap[userId] if !ok { - record = make([]int, 2) + record = make([]int, 4) record[0] = order record[1] = money + record[2] = week + record[3] = num bonusMap[userId] = record } else { if record[0] > order { record[0] = order record[1] = money + record[2] = week + record[3] = num } else { if record[0] == order && record[1] < money { record[1] = money + record[2] = week + record[3] = num } } } @@ -924,6 +931,17 @@ func getBonusWeekDataMap() map[int64][]int { return bonusMap } +func getWeekAndNum(name string) (int, int) { + name = name[0 : len(name)-4] + tmp := strings.Split(name, "_") + if len(tmp) == 2 { + week := getIntValue(tmp[0]) + num := getIntValue(tmp[1]) + return week, num + } + return 0, 0 +} + func getBonusMap() map[string]map[string]int { bonusMap := make(map[string]map[string]int) url := setting.RecommentRepoAddr + "bonus/record.txt" From d883eec2e51f099b970f1f6c2982583183ee695d Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 9 Jan 2023 16:34:36 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=A4=9A=E8=8A=82=E7=82=B9=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=B8=8B=E8=BD=BD=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 24 ++++++++++++++++++++---- routers/repo/modelarts.go | 20 ++++++++++++-------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 83b03ed44..3943a5f29 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -614,7 +614,7 @@ func ObsCreateObject(path string) error { return nil } -func GetObsLogFileName(prefix string) (string, error) { +func GetObsLogFileName(prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket input.Prefix = prefix @@ -622,10 +622,26 @@ func GetObsLogFileName(prefix string) (string, error) { output, err := ObsCli.ListObjects(input) if err != nil { log.Error("PutObject failed:", err.Error()) - return "", err + return nil, err } if output == nil || len(output.Contents) == 0 { - return "", errors.New("obs log files not exist") + return nil, errors.New("obs log files not exist") + } + fileInfos := make([]FileInfo, 0) + for _, val := range output.Contents { + //result[num] = c.Key + if strings.HasSuffix(val.Key, ".log") { + log.Info("log fileName=" + val.Key) + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: val.Key, + Size: val.Size, + IsDir: false, + ParenDir: "", + } + fileInfos = append(fileInfos, fileInfo) + } + } - return output.Contents[0].Key, nil + return fileInfos, nil } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index e0b9cd1b6..11f0b2e9b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2893,15 +2893,19 @@ func TrainJobDownloadLogFile(ctx *context.Context) { ctx.ServerError("GetObsLogFileName", err) return } - - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key) - if err != nil { - log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) - return + if len(key) > 1 { + ObsDownloadManyFile("", ctx, task.DisplayJobName+".zip", key) + } else { + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key[0].FileName) + if err != nil { + log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) + return + } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) } - ctx.Resp.Header().Set("Cache-Control", "max-age=0") - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) + } func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) { var isMultiDataset bool From 44bbf463434651f837a85a92979c13ab66991b7d Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 9 Jan 2023 17:05:28 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=A4=9A=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E8=AE=AD=E7=BB=83=E6=97=A5=E5=BF=97=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 4 ++-- routers/repo/modelarts.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 3943a5f29..cc621cc3c 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -634,10 +634,10 @@ func GetObsLogFileName(prefix string) ([]FileInfo, error) { log.Info("log fileName=" + val.Key) fileInfo := FileInfo{ ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: val.Key, + FileName: val.Key[len(prefix)-3:], //加上 job Size: val.Size, IsDir: false, - ParenDir: "", + ParenDir: prefix[0 : len(prefix)-3], } fileInfos = append(fileInfos, fileInfo) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 11f0b2e9b..7ffe5fef3 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2894,9 +2894,9 @@ func TrainJobDownloadLogFile(ctx *context.Context) { return } if len(key) > 1 { - ObsDownloadManyFile("", ctx, task.DisplayJobName+".zip", key) + ObsDownloadManyFile(prefix[0:len(prefix)-3], ctx, task.DisplayJobName+".zip", key) } else { - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key[0].FileName) + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key[0].ParenDir+key[0].FileName) if err != nil { log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err)