From c64f787fdf710d961c1d3657d7eb233041bfc06d Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 6 Jul 2022 11:08:52 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=89=93=E6=A6=9C=E6=95=B0=E6=8D=AE=E4=B8=8B=E8=BD=BD=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/repo_activity_custom.go | 36 +++ models/user_analysis_for_activity.go | 436 +++++++++++++++++++++++++++++++++++ models/user_business_analysis.go | 14 +- routers/api/v1/api.go | 1 + routers/repo/user_data_analysis.go | 108 +++++++++ 5 files changed, 589 insertions(+), 6 deletions(-) create mode 100644 models/user_analysis_for_activity.go diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index cbe00b9d9..6e7921d75 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -211,6 +211,42 @@ func setKeyContributerDict(contributorDistinctDict map[string]int, email string, } } +func GetAllUserPublicRepoKPIStats(startTime time.Time, endTime time.Time) (map[string]*git.UserKPIStats, error) { + authors := make(map[string]*git.UserKPIStats) + repositorys, err := GetAllRepositoriesByFilterCols("owner_name", "name", "is_private") + if err != nil { + return nil, err + } + + for _, repository := range repositorys { + if repository.IsPrivate { + continue + } + authorsOneRepo, err1 := git.GetUserKPIStats(repository.RepoPath(), startTime, endTime) + if err1 != nil { + log.Warn("get user kpi status err:"+repository.RepoPath(), err1.Error()) + continue + } + + for key, value := range authorsOneRepo { + if _, ok := authors[key]; !ok { + authors[key] = &git.UserKPIStats{ + + Name: value.Name, + Email: value.Email, + Commits: 0, + CommitLines: 0, + } + } + authors[key].Commits += value.Commits + authors[key].CommitLines += value.CommitLines + + } + + } + return authors, nil +} + func GetAllUserKPIStats(startTime time.Time, endTime time.Time) (map[string]*git.UserKPIStats, error) { authors := make(map[string]*git.UserKPIStats) repositorys, err := GetAllRepositoriesByFilterCols("owner_name", "name") diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go new file mode 100644 index 000000000..7de5f0df1 --- /dev/null +++ b/models/user_analysis_for_activity.go @@ -0,0 +1,436 @@ +package models + +import ( + "fmt" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" +) + +type UserBusinessAnalysisForActivity struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassigees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //use + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + + //user + Email string `xorm:"NOT NULL"` + + Phone string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` +} + +func QueryDataForActivity(startTime time.Time, endTime time.Time) []*UserBusinessAnalysisForActivity { + sess := x.NewSession() + defer sess.Close() + + result := make([]*UserBusinessAnalysisForActivity, 0) + publicRepo := queryPublicRepo() + start_unix := startTime.Unix() + end_unix := endTime.Unix() + + CodeMergeCountMap := queryPullRequestPublic(start_unix, end_unix, publicRepo) + CommitCodeSizeMap, err := GetAllUserPublicRepoKPIStats(startTime, endTime) + if err != nil { + log.Info("error,info=" + err.Error()) + } + CommitCountMap := queryCommitActionPublic(start_unix, end_unix, 5, publicRepo) + IssueCountMap, publicRepoIssueIdMap := queryCreateIssuePublic(start_unix, end_unix, publicRepo) + SolveIssueCountMap := querySolveIssuePublic(start_unix, end_unix, publicRepoIssueIdMap) + WatchedCountMap, _ := queryFollow(start_unix, end_unix) + CommentCountMap := queryCommentPublic(start_unix, end_unix, publicRepoIssueIdMap) + PublicDataSet := queryAllPublicDataSet(publicRepo) + DatasetFileNums := queryPublicDatasetFileNums(start_unix, end_unix, PublicDataSet) + AiModelManageMap := queryUserModelPublic(start_unix, end_unix, publicRepo) + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysisForActivity + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + + dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap) + dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap) + dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap) + dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, DatasetFileNums) + dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap) + dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap) + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } + dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap) + + dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + + result = append(result, &dateRecord) + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return result +} +func querySolveIssuePublic(start_unix int64, end_unix int64, publicRepoIssueIdMap map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := "issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Table("issue_assignees").Join("inner", "issue", "issue.id=issue_assignees.issue_id").Where(cond).Count(new(IssueAssignees)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where(cond).OrderBy("issue_assignees.id asc").Limit(PAGE_SIZE, int(indexTotal)) + + sess.Find(&issueAssigneesList) + + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if isPublicRepo(issueAssigneesRecord.IssueID, publicRepoIssueIdMap) { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return resultMap +} + +func queryPublicRepo() map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + + count, err := sess.Table("repository").Count(new(Repository)) + if err != nil { + log.Info("query Repository error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + repositoryList := make([]*Repository, 0) + sess.Select("*").Table("repository").OrderBy("id desc").Limit(PAGE_SIZE, int(indexTotal)) + sess.Find(&repositoryList) + log.Info("query repo size=" + fmt.Sprint(len(repositoryList))) + for _, repositoryRecord := range repositoryList { + if repositoryRecord.IsPrivate { + continue + } + if _, ok := resultMap[repositoryRecord.ID]; !ok { + resultMap[repositoryRecord.ID] = 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return resultMap +} + +func isPublicRepo(repoId int64, publicAllRepo map[int64]int) bool { + if _, ok := publicAllRepo[repoId]; !ok { + return false + } + return true +} + +func queryPullRequestPublic(start_unix int64, end_unix int64, publicAllRepo map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := "pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueList := make([]*Issue, 0) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(PAGE_SIZE, int(indexTotal)) + sess.Find(&issueList) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if isPublicRepo(issueRecord.RepoID, publicAllRepo) { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + +func queryCommitActionPublic(start_unix int64, end_unix int64, actionType int64, publicAllRepo map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + + cond := "user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id,repo_id").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if isPublicRepo(actionRecord.RepoID, publicAllRepo) { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return resultMap +} + +func queryCreateIssuePublic(start_unix int64, end_unix int64, publicAllRepo map[int64]int) (map[int64]int, map[int64]int) { + + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + publicRepoIssueIdMap := make(map[int64]int) + cond := "is_pull=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query Issue error. return.") + return resultMap, publicRepoIssueIdMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,poster_id,repo_id").Table("issue").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + issueList := make([]*Issue, 0) + sess.Find(&issueList) + log.Info("query issue size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if isPublicRepo(issueRecord.RepoID, publicAllRepo) { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + publicRepoIssueIdMap[issueRecord.ID] = 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap, publicRepoIssueIdMap + +} + +func queryCommentPublic(start_unix int64, end_unix int64, publicRepoIssueIdMap map[int64]int) map[int64]int { + + sess := x.NewSession() + defer sess.Close() + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + resultMap := make(map[int64]int) + count, err := sess.Where(cond).Count(new(Comment)) + if err != nil { + log.Info("query Comment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if isPublicRepo(commentRecord.IssueID, publicRepoIssueIdMap) { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + +func queryAllPublicDataSet(publicAllRepo map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + publicDataSetIdMap := make(map[int64]int) + count, err := sess.Count(new(Dataset)) + if err != nil { + log.Info("query dataset error. return.") + return publicDataSetIdMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,repo_id").Table(new(Dataset)).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + datasetList := make([]*Dataset, 0) + sess.Find(&datasetList) + log.Info("query datasetList size=" + fmt.Sprint(len(datasetList))) + for _, datasetRecord := range datasetList { + if isPublicRepo(datasetRecord.RepoID, publicAllRepo) { + publicDataSetIdMap[datasetRecord.ID] = 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return publicDataSetIdMap +} + +func queryPublicDatasetFileNums(start_unix int64, end_unix int64, publicDataSetIdMap map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultNumMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Attachment)) + if err != nil { + log.Info("query attachment error. return.") + return resultNumMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uploader_id,size,dataset_id").Table("attachment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if isPublicRepo(attachRecord.DatasetID, publicDataSetIdMap) { + if _, ok := resultNumMap[attachRecord.UploaderID]; !ok { + resultNumMap[attachRecord.UploaderID] = 1 + } else { + resultNumMap[attachRecord.UploaderID] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultNumMap +} + +func queryUserModelPublic(start_unix int64, end_unix int64, publicAllRepo map[int64]int) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelManage)) + if err != nil { + log.Info("query AiModelManage error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,repo_id").Table("ai_model_manage").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelManage, 0) + sess.Find(&aiModelList) + log.Info("query AiModelManage size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if isPublicRepo(aiModelRecord.RepoId, publicAllRepo) { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index e058c0df8..84b220f11 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1028,12 +1028,12 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, setUserMetrics(userMetrics, userRecord, start_unix, end_unix, dateRecord) if getUserActivate(dateRecord) > 0 { log.Info("has activity." + userRecord.Name) - addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) + addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID, currentTimeNow) } if userRecord.IsActive { - addUserToMap(userAcitvateJsonMap, userRecord.CreatedUnix, dateRecord.ID) + addUserToMap(userAcitvateJsonMap, userRecord.CreatedUnix, dateRecord.ID, currentTimeNow) } - addUserToMap(userCurrentDayRegistMap, userRecord.CreatedUnix, dateRecord.ID) + addUserToMap(userCurrentDayRegistMap, userRecord.CreatedUnix, dateRecord.ID, currentTimeNow) } indexTotal += PAGE_SIZE @@ -1056,7 +1056,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, useMetrics.NotActivateRegistUser = getMapKeyStringValue("NotActivateRegistUser", userMetrics) useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) - + useMetrics.CurrentDayRegistUser = getMapKeyStringValue("CurrentDayRegistUser", userMetrics) count, err = sess.Where("type=0").Count(new(User)) if err != nil { log.Info("query user error. return.") @@ -1124,8 +1124,9 @@ func setUniqueUserId(jsonString string, value map[int64]int64) (string, int) { return userIdArray, len(value) } -func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) { - CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location()) +func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64, currentTimeNow time.Time) { + registTime := registDate.AsTimeInLocation(currentTimeNow.Location()) + CountDateTime := time.Date(registDate.Year(), registTime.Month(), registTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) CountDate := CountDateTime.Unix() if _, ok := currentUserActivity[CountDate]; !ok { userIdMap := make(map[int64]int64, 0) @@ -1149,6 +1150,7 @@ func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, en } else { userMetrics["NotActivateRegistUser"] = getMapKeyStringValue("NotActivateRegistUser", userMetrics) + 1 } + userMetrics["CurrentDayRegistUser"] = getMapKeyStringValue("CurrentDayRegistUser", userMetrics) + 1 } if user.IsActive { userMetrics["TotalActivateRegistUser"] = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) + 1 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index e6c572e73..e086035f6 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -570,6 +570,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) + m.Get("/query_user_activity", operationReq, repo_ext.QueryUserActivity) //cloudbrain board m.Group("/cloudbrainboard", func() { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 207727af1..db0cea97c 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -721,3 +721,111 @@ func TimingCountData() { startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") TimingCountDataByDateAndReCount(startTime, false) } + +func QueryUserActivity(ctx *context.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + + t, _ := time.Parse("2006-01-02", startDate) + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + startTime = startTime.UTC() + + t, _ = time.Parse("2006-01-02", endDate) + endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location()) + endTime = endTime.UTC() + + sheetName := ctx.Tr("user.static.sheetname") + filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" + filePath := setting.AppDataPath + Excel_File_Path + filename + os.Remove(setting.AppDataPath + Excel_File_Path + filename) + + go writeUserActivityToExcel(startTime, endTime, filePath, ctx) + + ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+setting.AppURL+"api/v1/download_user_define_file?filename="+filename) + +} + +func writeUserActivityToExcel(startTime time.Time, endTime time.Time, filePath string, ctx *context.Context) { + re := models.QueryDataForActivity(startTime, endTime) + log.Info("return count=" + fmt.Sprint(len(re))) + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.static.sheetname") + index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + + excelHeader := make([]string, 0) + excelHeader = append(excelHeader, ctx.Tr("user.static.id")) + excelHeader = append(excelHeader, ctx.Tr("user.static.name")) + excelHeader = append(excelHeader, ctx.Tr("user.static.codemergecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.issuecount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commentcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.watchedcount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.commitcodesize")) + excelHeader = append(excelHeader, ctx.Tr("user.static.solveissuecount")) + + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) + excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) + excelHeader = append(excelHeader, ctx.Tr("user.static.email")) + excelHeader = append(excelHeader, ctx.Tr("user.static.registdate")) + + excelHeaderMap := make(map[string]string, 0) + var j byte + j = 0 + for _, value := range excelHeader { + excelColumn := getColumn(j) + fmt.Sprint(1) + log.Info("excelColumn=" + excelColumn) + excelHeaderMap[excelColumn] = value + j++ + } + + for k, v := range excelHeaderMap { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + row := i + 2 + rows := fmt.Sprint(row) + var tmp byte + tmp = 0 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ID) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Name) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CodeMergeCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.IssueCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommentCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.WatchedCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitCodeSize) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.SolveIssueCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitDatasetNum) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Email) + tmp = tmp + 1 + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3]) + tmp = tmp + 1 + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + + os.Mkdir(setting.AppDataPath+Excel_File_Path, 0755) + if err := xlsx.SaveAs(filePath); err != nil { + log.Info("writer exel error." + err.Error()) + } else { + log.Info("write to file succeed, filepath=" + filePath) + } +} From 5bfef49ffe0968b78b3e8b122c6fda520382bd67 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 11 Jul 2022 15:30:49 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E6=9C=BA?= =?UTF-8?q?=E5=8F=B7=E5=B9=B6=E8=B0=83=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_analysis_for_activity.go | 1 + models/user_business_analysis.go | 11 +++++++++-- models/user_business_struct.go | 14 ++++++++++++++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/user_data_analysis.go | 14 ++++++++++++-- 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index 7de5f0df1..f2c9e0a0c 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -81,6 +81,7 @@ func QueryDataForActivity(startTime time.Time, endTime time.Time) []*UserBusines dateRecord.ID = userRecord.ID log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) dateRecord.Email = userRecord.Email + dateRecord.Phone = userRecord.PhoneNumber dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 84b220f11..7e667d675 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -105,6 +105,8 @@ type UserBusinessAnalysisAll struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysis struct { @@ -192,6 +194,8 @@ type UserBusinessAnalysis struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisQueryOptions struct { @@ -475,6 +479,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi dateRecord.CountDate = CountDate.Unix() dateRecord.DataDate = DataDate dateRecord.Email = userRecord.Email + dateRecord.Phone = userRecord.PhoneNumber dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.UserLocation = userRecord.Location @@ -728,6 +733,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS var dateRecordAll UserBusinessAnalysisAll dateRecordAll.ID = userRecord.ID dateRecordAll.Email = userRecord.Email + dateRecordAll.Phone = userRecord.PhoneNumber dateRecordAll.RegistDate = userRecord.CreatedUnix dateRecordAll.Name = userRecord.Name dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) @@ -839,7 +845,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone) " + "VALUES" for i, record := range dateRecords { @@ -848,7 +854,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," + - fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ")" + fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "')" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -973,6 +979,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.CountDate = CountDate.Unix() dateRecord.Email = userRecord.Email + dateRecord.Phone = userRecord.PhoneNumber dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 870a64bc7..9cadbed0b 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -65,6 +65,8 @@ type UserBusinessAnalysisCurrentYear struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisLast30Day struct { @@ -130,6 +132,8 @@ type UserBusinessAnalysisLast30Day struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisLastMonth struct { @@ -195,6 +199,8 @@ type UserBusinessAnalysisLastMonth struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisCurrentMonth struct { @@ -260,6 +266,8 @@ type UserBusinessAnalysisCurrentMonth struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisCurrentWeek struct { @@ -326,6 +334,8 @@ type UserBusinessAnalysisCurrentWeek struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisYesterday struct { @@ -392,6 +402,8 @@ type UserBusinessAnalysisYesterday struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserBusinessAnalysisLastWeek struct { @@ -458,6 +470,8 @@ type UserBusinessAnalysisLastWeek struct { CollectImage int `xorm:"NOT NULL DEFAULT 0"` CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` + + Phone string `xorm:"NOT NULL"` } type UserAnalysisPara struct { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 2efaafdc9..0c9852c12 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -559,6 +559,7 @@ static.CollectImage=Collect Image Count static.CollectedImage=Collected Image Count static.RecommendImage=Recommended Image Count static.email=Email +static.phone=Phone static.location=Location static.all=All static.public.user_business_analysis_current_month=Current_Month diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index fe91aaed3..350d0800d 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -564,6 +564,7 @@ static.CollectImage=收藏镜像数 static.CollectedImage=被收藏镜像数 static.RecommendImage=被推荐镜像数 static.email=Email +static.phone=电话 static.location=所在地区 static.all=所有 static.public.user_business_analysis_current_month=本月 diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index db0cea97c..cc8d3dced 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -112,6 +112,7 @@ func getExcelHeader(ctx *context.Context) map[string]string { excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendImage")) excelHeader = append(excelHeader, ctx.Tr("user.static.email")) + excelHeader = append(excelHeader, ctx.Tr("user.static.phone")) excelHeader = append(excelHeader, ctx.Tr("user.static.location")) excelHeader = append(excelHeader, ctx.Tr("user.static.registdate")) @@ -193,6 +194,9 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Email) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Phone) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.UserLocation) tmp = tmp + 1 @@ -268,6 +272,9 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord * xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Email) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Phone) + tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.UserLocation) tmp = tmp + 1 @@ -723,8 +730,8 @@ func TimingCountData() { } func QueryUserActivity(ctx *context.Context) { - startDate := ctx.Query("startDate") - endDate := ctx.Query("endDate") + startDate := ctx.Query("beginTime") + endDate := ctx.Query("endTime") t, _ := time.Parse("2006-01-02", startDate) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -768,6 +775,7 @@ func writeUserActivityToExcel(startTime time.Time, endTime time.Time, filePath s excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) excelHeader = append(excelHeader, ctx.Tr("user.static.email")) + excelHeader = append(excelHeader, ctx.Tr("user.static.phone")) excelHeader = append(excelHeader, ctx.Tr("user.static.registdate")) excelHeaderMap := make(map[string]string, 0) @@ -814,6 +822,8 @@ func writeUserActivityToExcel(startTime time.Time, endTime time.Time, filePath s tmp = tmp + 1 xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Email) tmp = tmp + 1 + xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Phone) + tmp = tmp + 1 formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3]) tmp = tmp + 1 From 9d0557d8acf0b38124aac646d4e3a8bc91efe292 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 11 Jul 2022 15:37:25 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=94=B5=E8=AF=9D?= =?UTF-8?q?=E5=8F=B7=E7=A0=81=E7=9A=84=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_analysis_for_activity.go | 2 +- models/user_business_struct.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index f2c9e0a0c..d241fa72c 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -32,7 +32,7 @@ type UserBusinessAnalysisForActivity struct { //user Email string `xorm:"NOT NULL"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 9cadbed0b..36ef077e2 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -66,7 +66,7 @@ type UserBusinessAnalysisCurrentYear struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisLast30Day struct { @@ -133,7 +133,7 @@ type UserBusinessAnalysisLast30Day struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisLastMonth struct { @@ -200,7 +200,7 @@ type UserBusinessAnalysisLastMonth struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisCurrentMonth struct { @@ -267,7 +267,7 @@ type UserBusinessAnalysisCurrentMonth struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisCurrentWeek struct { @@ -335,7 +335,7 @@ type UserBusinessAnalysisCurrentWeek struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisYesterday struct { @@ -403,7 +403,7 @@ type UserBusinessAnalysisYesterday struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisLastWeek struct { @@ -471,7 +471,7 @@ type UserBusinessAnalysisLastWeek struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserAnalysisPara struct { From 314e2df169e35df2441f4124abb327df7db253e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 11 Jul 2022 15:41:02 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=94=B5=E8=AF=9D?= =?UTF-8?q?=E5=8F=B7=E7=A0=81=E7=9A=84=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 7e667d675..18f7fa6b2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -106,7 +106,7 @@ type UserBusinessAnalysisAll struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysis struct { @@ -195,7 +195,7 @@ type UserBusinessAnalysis struct { CollectedImage int `xorm:"NOT NULL DEFAULT 0"` RecommendImage int `xorm:"NOT NULL DEFAULT 0"` - Phone string `xorm:"NOT NULL"` + Phone string `xorm:"NULL"` } type UserBusinessAnalysisQueryOptions struct { From 2cf8596fe571814aa5026c69fa57604c46268a77 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 11 Jul 2022 15:49:53 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=B8=8B=E8=BD=BD=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index cc8d3dced..a83017f7b 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -608,7 +608,7 @@ func QueryUserStaticDataPage(ctx *context.Context) { filename := sheetName + "_" + startDate + "_" + endDate + ".xlsx" os.Remove(setting.AppDataPath + Excel_File_Path + filename) go writeFileToDisk(ctx, count, re, filename) - ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+setting.AppURL+"api/v1/download_user_define_file?filename="+filename) + ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+"/api/v1/download_user_define_file?filename="+filename) } else { mapInterface := make(map[string]interface{}) re, count := models.QueryUserStaticDataPage(pageOpts) From 8cfbb00ca0902bedbf0eb2aacebf39dabb13aa67 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 11 Jul 2022 15:50:26 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=B8=8B=E8=BD=BD=E8=B7=AF=E5=BE=84=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index a83017f7b..ac1265d04 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -748,7 +748,7 @@ func QueryUserActivity(ctx *context.Context) { go writeUserActivityToExcel(startTime, endTime, filePath, ctx) - ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+setting.AppURL+"api/v1/download_user_define_file?filename="+filename) + ctx.JSON(http.StatusOK, ctx.Tr("user.static.downloadinfo")+"/api/v1/download_user_define_file?filename="+filename) } From 79c04cc0da856eeb1526d011585765fcfbd1360d Mon Sep 17 00:00:00 2001 From: avadesian Date: Wed, 13 Jul 2022 09:03:39 +0800 Subject: [PATCH 07/15] change cloudbrain debug command --- modules/cloudbrain/cloudbrain.go | 9 +++++++-- modules/setting/setting.go | 4 ++++ routers/repo/cloudbrain.go | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 430304dd5..7fca45401 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -17,7 +17,7 @@ import ( ) const ( - Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + //Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" @@ -70,6 +70,11 @@ type GenerateCloudBrainTaskReq struct { ResourceSpecId int } +func GetCloudbrainDebugCommand() string { + var command = `pip3 install jupyterlab==3 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;/usr/local/bin/python /usr/local/bin/jupyter-lab --ServerApp.shutdown_no_activity_timeout=` + setting.CullIdleTimeout + ` --TerminalManager.cull_inactive_timeout=` + setting.CullIdleTimeout + ` --TerminalManager.cull_interval=` + setting.CullInterval + ` --MappingKernelManager.cull_idle_timeout=` + setting.CullIdleTimeout + ` --MappingKernelManager.cull_interval=` + setting.CullInterval + ` --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --ServerApp.token="" --ServerApp.allow_origin="self https://cloudbrain.pcl.ac.cn" ` + return command +} + func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { if !ctx.IsSigned { return false @@ -486,7 +491,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e GPUNumber: resourceSpec.GpuNum, MemoryMB: resourceSpec.MemMiB, ShmMB: resourceSpec.ShareMemMiB, - Command: Command, + Command: GetCloudbrainDebugCommand(),//Command, NeedIBDevice: false, IsMainRole: false, UseNNI: false, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2058c51a8..f429cc75d 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -466,6 +466,8 @@ var ( TrainGpuTypes string TrainResourceSpecs string MaxDatasetNum int + CullIdleTimeout string + CullInterval string //benchmark config IsBenchmarkEnabled bool @@ -1312,6 +1314,8 @@ func NewContext() { TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) + CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") + CullInterval = sec.Key("CULL_INTERVAL").MustString("60") sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5a3d0a6f8..59cccd9af 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -135,7 +135,7 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } ctx.Data["attachments"] = attachs - ctx.Data["command"] = cloudbrain.Command + ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand() ctx.Data["code_path"] = cloudbrain.CodeMountPath ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath ctx.Data["model_path"] = cloudbrain.ModelMountPath @@ -269,7 +269,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - command := cloudbrain.Command + command := cloudbrain.GetCloudbrainDebugCommand() if jobType == string(models.JobTypeTrain) { tpl = tplCloudBrainTrainJobNew commandTrain, err := getTrainJobCommand(form) @@ -2094,7 +2094,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) repo := ctx.Repo.Repository tpl := tplCloudBrainBenchmarkNew - command := cloudbrain.Command + command := cloudbrain.GetCloudbrainDebugCommand() tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { From 28bc2cc84cdbd96c62ecb1cbfe15d2b8a32d4dc5 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 09:52:45 +0800 Subject: [PATCH 08/15] #2373 fix bug --- routers/api/v1/repo/modelarts.go | 42 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 2a0ce19db..751e240bf 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -7,8 +7,10 @@ package repo import ( "code.gitea.io/gitea/modules/grampus" + "code.gitea.io/gitea/modules/setting" "encoding/json" "net/http" + "path" "strconv" "strings" @@ -263,39 +265,49 @@ func TrainJobGetLog(ctx *context.APIContext) { return } - resultLogFile, result, err := trainJobGetLogContent(jobID, versionName, baseLine, order, lines_int) + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + return + } + resultLogFile, result, err := trainJobGetLogContent(jobID, task.VersionID, baseLine, order, lines_int) if err != nil { log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) return } + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, modelarts.LogPath, versionName), "/") + "/job" + _, err = storage.GetObsLogFileName(prefix) + var canLogDownload bool + if err != nil { + canLogDownload = false + } else { + canLogDownload = true + } + ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "LogFileName": resultLogFile.LogFileList[0], - "StartLine": result.StartLine, - "EndLine": result.EndLine, - "Content": result.Content, - "Lines": result.Lines, + "JobID": jobID, + "LogFileName": resultLogFile.LogFileList[0], + "StartLine": result.StartLine, + "EndLine": result.EndLine, + "Content": result.Content, + "Lines": result.Lines, + "CanLogDownload": canLogDownload, }) } -func trainJobGetLogContent(jobID string, versionName string, baseLine string, order string, lines int) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) { - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) - return nil, nil, err - } +func trainJobGetLogContent(jobID string, versionID int64, baseLine string, order string, lines int) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) { - resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(versionID, 10)) if err != nil { log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error()) return nil, nil, err } - result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines) + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(versionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines) if err != nil { log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) return nil, nil, err From 581670cf372572c10811be0816193ffb192ffbde Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 13 Jul 2022 16:46:22 +0800 Subject: [PATCH 09/15] #2225 add MB2GB --- modules/templates/helper.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index dbb9354aa..857e365f8 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -18,6 +18,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" texttmpl "text/template" "time" @@ -327,6 +328,7 @@ func NewFuncMap() []template.FuncMap { }, "GetRefType": GetRefType, "GetRefName": GetRefName, + "MB2GB": MB2GB, }} } @@ -785,3 +787,14 @@ func GetRefName(ref string) string { reg := regexp.MustCompile(REF_TYPE_PATTERN) return reg.ReplaceAllString(ref, "") } + +func MB2GB(size int64) string { + s := strconv.FormatFloat(float64(size)/float64(1024), 'f', 2, 64) + for strings.HasSuffix(s, "0") { + s = strings.TrimSuffix(s, "0") + } + if strings.HasSuffix(s, ".") { + s = strings.TrimSuffix(s, ".") + } + return s +} From 7ff690e7aacbd7a17d022f0f583960ed06171996 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 14 Jul 2022 09:28:08 +0800 Subject: [PATCH 10/15] fix issue --- templates/repo/grampus/trainjob/show.tmpl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 579e83693..18c9cc427 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -480,8 +480,13 @@
-
+ {{if eq .ComputeResource "CPU/GPU"}} +
+ + {{$.i18n.Tr "repo.file_limit_100"}} +
+ {{end}} From f6230e4e1e91a3eb0f9a844d79ff1197f55f8b4d Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 14 Jul 2022 09:31:00 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=88=86=E6=9E=90?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E6=9C=BA=E5=8F=B7=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index b8d48b67d..c96e84a3d 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -226,8 +226,8 @@ width="120px" align="center"> - - + + Date: Thu, 14 Jul 2022 09:31:17 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=88=86=E6=9E=90?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E6=9C=BA=E5=8F=B7=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index c96e84a3d..fc87401ac 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -226,7 +226,7 @@ width="120px" align="center"> - + Date: Thu, 14 Jul 2022 09:49:34 +0800 Subject: [PATCH 13/15] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index bd542d912..9c2a91921 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -488,7 +488,7 @@
{{$.i18n.Tr "repo.modelarts.download_log"}} From d71473ec30d6392d8a21865b9344459f8d39beac Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 14 Jul 2022 09:57:28 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=88=86=E6=9E=90?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E6=9C=BA=E5=8F=B7=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- web_src/js/components/UserAnalysis.vue | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index fc87401ac..ef62e1d32 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -226,8 +226,11 @@ width="120px" align="center"> - - + Date: Thu, 14 Jul 2022 11:23:35 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=88=86=E6=9E=90?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E6=9C=BA=E5=8F=B7=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 18f7fa6b2..6471636e3 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1133,7 +1133,7 @@ func setUniqueUserId(jsonString string, value map[int64]int64) (string, int) { func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64, currentTimeNow time.Time) { registTime := registDate.AsTimeInLocation(currentTimeNow.Location()) - CountDateTime := time.Date(registDate.Year(), registTime.Month(), registTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + CountDateTime := time.Date(registTime.Year(), registTime.Month(), registTime.Day(), 0, 1, 0, 0, currentTimeNow.Location()) CountDate := CountDateTime.Unix() if _, ok := currentUserActivity[CountDate]; !ok { userIdMap := make(map[int64]int64, 0)