用户年度统计报告信息更改。

Signed-off-by: zouap <zouap@pcl.ac.cn>

models/user_business_analysis.go

@@ -586,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 	startTime := currentTimeNow.AddDate(0, 0, -1)
 
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
-	CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
+	CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
 
 	CommentCountMap := queryComment(start_unix, end_unix)
@@ -602,9 +602,9 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 		//log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson))
 	}
 	//CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix())
-	CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
+	CommitDatasetSizeMap, CommitDatasetNumMap, _ := queryDatasetSize(start_unix, end_unix)
 	SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
-	CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)
+	CreateRepoCountMap, _, _ := queryUserCreateRepo(start_unix, end_unix)
 	LoginCountMap := queryLoginCount(start_unix, end_unix)
 
 	OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix)
@@ -613,19 +613,14 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 	AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)
 
 	CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
-	RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)
+	RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
 	CollectImage, CollectedImage := queryImageStars(start_unix, end_unix)
 	RecommendImage := queryRecommedImage(start_unix, end_unix)
 
 	InvitationMap := queryUserInvitationCount(start_unix, end_unix)
 
 	DataDate := currentTimeNow.Format("2006-01-02") + " 00:01"
-	bonusMap := make(map[string]map[string]int)
-	if isUserYearData(tableName) {
-		bonusMap = getBonusMap()
-		log.Info("truncate all data from table:user_summary_current_year ")
-		statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")
-	}
+
 	cond := "type != 1 and is_active=true"
 	count, err := sess.Where(cond).Count(new(User))
 	if err != nil {
@@ -724,37 +719,6 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 					userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1
 				}
 			}
-			if isUserYearData(tableName) {
-				//年度数据
-				subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
-				mostActiveDay := ""
-				if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
-					mostActiveDay = getMostActiveJson(userInfo)
-				}
-				scoreMap := make(map[string]float64)
-				repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
-				dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
-				scoreMap["datasetscore"] = datasetscore
-				codeInfo, codescore := getCodeInfo(dateRecordAll)
-				scoreMap["codescore"] = codescore
-				cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
-				playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
-				re := &UserSummaryCurrentYear{
-					ID:             dateRecordAll.ID,
-					Name:           dateRecordAll.Name,
-					Email:          dateRecordAll.Email,
-					Phone:          dateRecordAll.Phone,
-					RegistDate:     dateRecordAll.RegistDate,
-					DateCount:      int(subTime.Hours()) / 24,
-					MostActiveDay:  mostActiveDay,
-					RepoInfo:       repoInfo,
-					DataSetInfo:    dataSetInfo,
-					CodeInfo:       codeInfo,
-					CloudBrainInfo: cloudBrainInfo,
-					PlayARoll:      playARoll,
-				}
-				statictisSess.Insert(re)
-			}
 		}
 		if len(dateRecordBatch) > 0 {
 			err := insertTable(dateRecordBatch, tableName, statictisSess)
@@ -784,6 +748,127 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
 	log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount))
 }
 
+func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) {
+	sess := x.NewSession()
+	defer sess.Close()
+	log.Info("RefreshUserYearTable start....")
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+
+	log.Info("UserYear StartTime:" + pageStartTime.Format("2006-01-02 15:04:05"))
+	log.Info("UserYear EndTime time:" + pageEndTime.Format("2006-01-02 15:04:05"))
+
+	start_unix := pageStartTime.Unix()
+	end_unix := pageEndTime.Unix()
+
+	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
+	CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
+	IssueCountMap := queryCreateIssue(start_unix, end_unix)
+
+	CommentCountMap := queryComment(start_unix, end_unix)
+
+	CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime)
+	if err != nil {
+		log.Info("query commit code errr.")
+	} else {
+		log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap)))
+	}
+	CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
+	SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
+	CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)
+
+	CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
+
+	_, CollectedDataset := queryDatasetStars(start_unix, end_unix)
+	_, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)
+
+	bonusMap := getBonusMap()
+	log.Info("truncate all data from table:user_summary_current_year ")
+	statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")
+
+	cond := "type != 1 and is_active=true"
+	count, err := sess.Where(cond).Count(new(User))
+	if err != nil {
+		log.Info("query user error. return.")
+		return
+	}
+	var indexTotal int64
+	indexTotal = 0
+	for {
+		sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
+		userList := make([]*User, 0)
+		sess.Find(&userList)
+		for _, userRecord := range userList {
+			var dateRecordAll UserBusinessAnalysisAll
+			dateRecordAll.ID = userRecord.ID
+			dateRecordAll.Email = userRecord.Email
+			dateRecordAll.Phone = userRecord.PhoneNumber
+			dateRecordAll.RegistDate = userRecord.CreatedUnix
+			dateRecordAll.Name = userRecord.Name
+
+			dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap)
+			dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap)
+			dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap)
+			dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap)
+
+			if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok {
+				dateRecordAll.CommitCodeSize = 0
+			} else {
+				dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines)
+			}
+			//dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap)
+			dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap)
+			dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap)
+			dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap)
+			dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap)
+
+			dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap)
+			dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap)
+			dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap)
+			dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap)
+			dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap)
+			dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap)
+			dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
+			dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
+
+			//年度数据
+			subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
+			mostActiveDay := ""
+			if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
+				mostActiveDay = getMostActiveJson(userInfo)
+			}
+			scoreMap := make(map[string]float64)
+			repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
+			dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
+			scoreMap["datasetscore"] = datasetscore
+			codeInfo, codescore := getCodeInfo(dateRecordAll)
+			scoreMap["codescore"] = codescore
+			cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
+			playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
+			re := &UserSummaryCurrentYear{
+				ID:             dateRecordAll.ID,
+				Name:           dateRecordAll.Name,
+				Email:          dateRecordAll.Email,
+				Phone:          dateRecordAll.Phone,
+				RegistDate:     dateRecordAll.RegistDate,
+				DateCount:      int(subTime.Hours()) / 24,
+				MostActiveDay:  mostActiveDay,
+				RepoInfo:       repoInfo,
+				DataSetInfo:    dataSetInfo,
+				CodeInfo:       codeInfo,
+				CloudBrainInfo: cloudBrainInfo,
+				PlayARoll:      playARoll,
+			}
+			statictisSess.Insert(re)
+		}
+		indexTotal += PAGE_SIZE
+		if indexTotal >= count {
+			break
+		}
+	}
+	log.Info("update user year data finished. ")
+}
+
 func isUserYearData(tableName string) bool {
 	if tableName == "user_business_analysis_current_year" {
 		currentTimeNow := time.Now()
@@ -1504,10 +1589,6 @@ func getInt(str string) int {
 	return int(re)
 }
 
-func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
-	CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
-}
-
 func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
 	sess := x.NewSession()
 	defer sess.Close()

routers/repo/cloudbrain.go

@@ -711,7 +711,6 @@ func CloudBrainRestart(ctx *context.Context) {
 	})
 }
 
-
 func getOldJobPath(task *models.Cloudbrain) string {
 	return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName
 }
@@ -854,10 +853,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
 func CloudBrainDebug(ctx *context.Context) {
 	task := ctx.Cloudbrain
 	debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
-	if task.BootFile!=""{
-		ctx.Redirect(getFileUrl(debugUrl,task.BootFile))
+	if task.BootFile != "" {
+		ctx.Redirect(getFileUrl(debugUrl, task.BootFile))
 
-	}else{
+	} else {
 		ctx.Redirect(debugUrl)
 	}
 
@@ -1581,6 +1580,21 @@ func CloudBrainDownloadModel(ctx *context.Context) {
 	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
+
+func CloudBrainDownloadMultiModel(ctx *context.Context) {
+	parentDir := ctx.Query("parentDir")
+	jobName := ctx.Query("jobName")
+	filePath := "jobs/" + jobName + "/model/" + parentDir
+	allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
+	if err == nil {
+		returnFileName := jobName + ".zip"
+		MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
+	} else {
+		log.Info("error,msg=" + err.Error())
+		ctx.ServerError("no file to download.", err)
+	}
+}
+
 func CloudBrainDownloadInferenceResult(ctx *context.Context) {
 	parentDir := ctx.Query("parentDir")
 	fileName := ctx.Query("fileName")

routers/repo/modelarts.go

@@ -2582,6 +2582,44 @@ func InferenceJobShow(ctx *context.Context) {
 	ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow)
 }
 
+func MultiModelDownload(ctx *context.Context) {
+	var (
+		err error
+	)
+	jobID := ctx.Params(":jobid")
+	versionName := ctx.Query("version_name")
+	parentDir := ctx.Query("parent_dir")
+
+	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
+	if err != nil {
+		log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error())
+		return
+	}
+
+	if task.ComputeResource == models.NPUResource {
+		path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/")
+		allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
+		if err == nil {
+			returnFileName := task.DisplayJobName + ".zip"
+			ObsDownloadManyFile(path, ctx, returnFileName, allFile)
+		} else {
+			log.Info("error,msg=" + err.Error())
+			ctx.ServerError("no file to download.", err)
+		}
+	} else if task.ComputeResource == models.GPUResource {
+		filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir
+		allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
+		if err == nil {
+			returnFileName := task.DisplayJobName + ".zip"
+			MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
+		} else {
+			log.Info("error,msg=" + err.Error())
+			ctx.ServerError("no file to download.", err)
+		}
+	}
+
+}
+
 func ModelDownload(ctx *context.Context) {
 	var (
 		err error

routers/repo/user_data_analysis.go

@@ -21,6 +21,7 @@ import (
 const (
 	PAGE_SIZE       = 2000
 	Excel_File_Path = "/useranalysis/"
+	USER_YEAR       = 2022
 )
 
 func getUserMetricsExcelHeader(ctx *context.Context) map[string]string {
@@ -719,6 +720,12 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) {
 	log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05"))
 	log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05"))
 	warnEmailMessage := "用户统计信息入库失败，请尽快定位。"
+
+	startYear := time.Date(USER_YEAR, 0, 0, 0, 0, 0, 1, t.Location())
+	endYear := startYear.AddDate(1, 0, 0)
+
+	models.RefreshUserYearTable(startYear, endYear)
+
 	//query wiki data
 	log.Info("start to time count data")
 	wikiMap, err := queryWikiCountMap(startTime, endTime)

routers/routes/routes.go

@@ -1175,6 +1175,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate)
 				m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
 				m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel)
+				m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 			})
 			m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew)
 			m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
@@ -1198,6 +1199,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel)
 					//m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
 					m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel)
+					m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 					//m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir)
 					//m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
 					m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew)
@@ -1210,7 +1212,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Group("/:jobid", func() {
 					m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow)
 					m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult)
-
+					m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 					m.Get("/downloadall", repo.DownloadInferenceResultFile)
 				})
 				m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew)