diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index af96444ac..ace51002d 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -11,30 +11,30 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string `xorm:"NULL"` + ID string `xorm:"pk"` + Name string `xorm:"INDEX NOT NULL"` + Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` + New int `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + AttachmentId string `xorm:"NULL"` + RepoId int64 `xorm:"INDEX NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + UserName string UserRelAvatarLink string `xorm:"NULL"` TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` IsCanOper bool } @@ -197,7 +197,6 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { Find(&aiModelManages); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } - sess.Close() return aiModelManages, count, nil } diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6e8ee1505..ceb552811 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1110,7 +1110,8 @@ func UpdateJob(job *Cloudbrain) error { func updateJob(e Engine, job *Cloudbrain) error { var sess *xorm.Session sess = e.Where("job_id = ?", job.JobID) - _, err := sess.Cols("status", "container_id", "container_ip").Update(job) + //_, err := sess.Cols("status", "container_id", "container_ip").Update(job) + _, err := sess.Update(job) return err } diff --git a/models/dataset.go b/models/dataset.go index 402a548ef..2b3de752b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -93,6 +93,7 @@ type SearchDatasetOptions struct { IncludePublic bool ListOptions SearchOrderBy + IsOwner bool } func CreateDataset(dataset *Dataset) (err error) { @@ -150,6 +151,9 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) + if !opts.IsOwner { + cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) + } } return cond diff --git a/models/repo.go b/models/repo.go index 7948346c5..fda5460f5 100755 --- a/models/repo.go +++ b/models/repo.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "html/template" + "xorm.io/xorm" "code.gitea.io/gitea/modules/blockchain" @@ -1632,6 +1633,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error { releaseAttachments = append(releaseAttachments, attachments[i].RelativePath()) } + // Delete dataset attachment record and remove related files + deleteDatasetAttachmentByRepoId(sess, repoID) + if err = deleteBeans(sess, &Access{RepoID: repo.ID}, &Action{RepoID: repo.ID}, @@ -1817,6 +1821,20 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return nil } +func deleteDatasetAttachmentByRepoId(sess *xorm.Session, repoId int64) error { + attachments := make([]*Attachment, 0) + if err := sess.Join("INNER", "dataset", "dataset.id = attachment.dataset_id"). + Where("dataset.repo_id = ?", repoId). + Find(&attachments); err != nil { + return err + } + if len(attachments) == 0 { + return nil + } + _, err := DeleteAttachments(attachments, true) + return err +} + // GetRepositoryByOwnerAndName returns the repository by given ownername and reponame. func GetRepositoryByOwnerAndName(ownerName, repoName string) (*Repository, error) { return getRepositoryByOwnerAndName(x, ownerName, repoName) diff --git a/models/user.go b/models/user.go index ec96420dc..b362472e8 100755 --- a/models/user.go +++ b/models/user.go @@ -929,7 +929,7 @@ var ( "template", "user", "vendor", - "dashbord", + "dashboard", "operation", "blockchain", "avatar", diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 15f67e858..9b9d5e0ad 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -7,10 +7,14 @@ import ( "strconv" "time" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" - "xorm.io/xorm" +) + +const ( + Page_SIZE = 2000 ) type UserBusinessAnalysisAll struct { @@ -199,7 +203,13 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) if allCount == 0 { - RefreshUserStaticAllTabel() + CommitCodeSizeMap, err := GetAllUserKPIStats() + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + RefreshUserStaticAllTabel(make(map[string]int), CommitCodeSizeMap) } pageSize := 1000 totalPage := int(allCount) / pageSize @@ -336,47 +346,164 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel() { +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + + sess := x.NewSession() + defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("delete all data from table: user_business_analysis_all") - statictisSess.Exec("delete from user_business_analysis_all") + log.Info("truncate all data from table: user_business_analysis_all") + statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() - var cond = builder.NewCond() - cond = cond.And( - builder.Gte{"count_date": pageStartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": pageEndTime}, - ) - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). - Find(&userBusinessAnalysisList); err != nil { + startTime := currentTimeNow.AddDate(0, 0, -1) + + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryAction(start_unix, end_unix, 6) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap := queryFollow(start_unix, end_unix) + + CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) + + DataDate := currentTimeNow.Format("2006-01-02") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") return } - log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) - allData := getAllData(userRecord.ID, statictisSess) - allData.ID = userRecord.ID - allData.CountDate = 0 - allData.DataDate = userRecord.DataDate - allData.Email = userRecord.Email - allData.OpenIIndex = userRecord.OpenIIndex - allData.GiteaAgeMonth = userRecord.GiteaAgeMonth - allData.Name = userRecord.Name - allData.RegistDate = userRecord.RegistDate - - _, err := statictisSess.Insert(&allData) - if err != nil { - log.Info("insert all data failed." + err.Error()) + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for i, userRecord := range userList { + log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecordAll.DataDate = DataDate + + if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CodeMergeCount = 0 + } else { + dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitCount = 0 + } else { + dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] + } + + if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.IssueCount = 0 + } else { + dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] + } + + if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommentCount = 0 + } else { + dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] + } + + if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.FocusRepoCount = 0 + } else { + dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] + } + + if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.StarRepoCount = 0 + } else { + dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] + } + + if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.WatchedCount = 0 + } else { + dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + + if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitDatasetSize = 0 + } else { + dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] + } + + if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.SolveIssueCount = 0 + } else { + dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] + } + + if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { + dateRecordAll.EncyclopediasCount = 0 + } else { + dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] + } + + if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CreateRepoCount = 0 + } else { + dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] + } + + if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.LoginCount = 0 + } else { + dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] + } + + if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { + dateRecordAll.OpenIIndex = 0 + } else { + dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] + } + + dateRecordAll.CommitModelCount = 0 + _, err = statictisSess.Insert(&dateRecordAll) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + log.Info("refresh all data finished.") } @@ -385,19 +512,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") - userList := make([]*User, 0) - sess.Find(&userList) currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - //yesterday := currentTimeNow.AddDate(0, 0, -1) - //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) if isReCount { @@ -429,193 +550,134 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, statictisSess := xStatistic.NewSession() defer statictisSess.Close() - for i, userRecord := range userList { - var dateRecord UserBusinessAnalysis - dateRecord.ID = userRecord.ID - log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - dateRecord.CountDate = CountDate.Unix() - - statictisSess.Delete(&dateRecord) - - dateRecord.Email = userRecord.Email - dateRecord.RegistDate = userRecord.CreatedUnix - dateRecord.Name = userRecord.Name - dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) - dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return err + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate + if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { + dateRecord.CodeMergeCount = 0 + } else { + dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] + } - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } + if _, ok := CommitCountMap[dateRecord.ID]; !ok { + dateRecord.CommitCount = 0 + } else { + dateRecord.CommitCount = CommitCountMap[dateRecord.ID] + } - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } + if _, ok := IssueCountMap[dateRecord.ID]; !ok { + dateRecord.IssueCount = 0 + } else { + dateRecord.IssueCount = IssueCountMap[dateRecord.ID] + } - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } + if _, ok := CommentCountMap[dateRecord.ID]; !ok { + dateRecord.CommentCount = 0 + } else { + dateRecord.CommentCount = CommentCountMap[dateRecord.ID] + } - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } + if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { + dateRecord.FocusRepoCount = 0 + } else { + dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] + } - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { + dateRecord.StarRepoCount = 0 + } else { + dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] + } - if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { - dateRecord.CommitCodeSize = 0 - } else { - dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) - } + if _, ok := WatchedCountMap[dateRecord.ID]; !ok { + dateRecord.WatchedCount = 0 + } else { + dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] + } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { + dateRecord.CommitDatasetSize = 0 + } else { + dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] + } - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { + dateRecord.SolveIssueCount = 0 + } else { + dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] + } - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } + if _, ok := wikiCountMap[dateRecord.Name]; !ok { + dateRecord.EncyclopediasCount = 0 + } else { + dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] + } - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } - if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { - dateRecord.OpenIIndex = 0 - } else { - dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] - } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } - dateRecord.CommitModelCount = 0 + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } - _, err = statictisSess.Insert(&dateRecord) - if err != nil { - log.Info("insert daterecord failed." + err.Error()) - return err - } + dateRecord.CommitModelCount = 0 - if isExistUserInAllTable(dateRecord.ID, statictisSess) { - updateCurrentData(dateRecord.ID, statictisSess, dateRecord) - } else { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) - allData := getAllData(dateRecord.ID, statictisSess) - allData.ID = dateRecord.ID - allData.CountDate = 0 - allData.DataDate = dateRecord.DataDate - allData.Email = dateRecord.Email - allData.OpenIIndex = dateRecord.OpenIIndex - allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth - allData.Name = dateRecord.Name - allData.RegistDate = dateRecord.RegistDate - - _, err = statictisSess.Insert(&allData) + _, err = statictisSess.Insert(&dateRecord) if err != nil { - log.Info("insert all data failed." + err.Error()) + log.Info("insert daterecord failed." + err.Error()) return err } } + indexTotal += Page_SIZE + if indexTotal >= count { + break + } } - return nil -} - -func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { - - _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + - ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + - ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + - ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + - ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + - ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + - ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + - ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + - ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + - ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + - ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + - ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + - ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + - ",login_count+=" + fmt.Sprint(currentData.LoginCount) + - " where id=" + fmt.Sprint(userId)) - - if err != nil { - log.Info("update table failed." + err.Error()) - } - -} -func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { - - allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) - if err != nil { - return false - } - return allCount > 0 -} + RefreshUserStaticAllTabel(wikiCountMap, CommitCodeSizeMap) -func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { - var dateRecord UserBusinessAnalysisAll - - rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") - if err == nil { - for i, row := range rows { - log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) - dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) - dateRecord.CommitCount = getInt(string(row["commit_count"])) - dateRecord.IssueCount = getInt(string(row["issue_count"])) - dateRecord.CommentCount = getInt(string(row["comment_count"])) - dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) - dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) - dateRecord.WatchedCount = getInt(string(row["watched_count"])) - dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) - dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) - dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) - dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) - dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) - dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) - dateRecord.LoginCount = getInt(string(row["login_count"])) - } - } - return dateRecord + return nil } func getInt(str string) int { @@ -631,44 +693,70 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { - //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() defer sess.Close() - sess.Select("issue_assignees.*").Table("issue_assignees"). - Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) - issueAssigneesList := make([]*IssueAssignees, 0) - sess.Find(&issueAssigneesList) resultMap := make(map[int64]int) - log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) - for _, issueAssigneesRecord := range issueAssigneesList { - if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { - resultMap[issueAssigneesRecord.AssigneeID] = 1 - } else { - resultMap[issueAssigneesRecord.AssigneeID] += 1 + cond := "issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Table("issue_assignees").Join("inner", "issue", "issue.id=issue_assignees.issue_id").Where(cond).Count(new(IssueAssignees)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where(cond).Limit(Page_SIZE, int(indexTotal)) + + sess.Find(&issueAssigneesList) + + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - - sess.Select("issue.*").Table("issue"). - Join("inner", "pull_request", "issue.id=pull_request.issue_id"). - Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) - - issueList := make([]*Issue, 0) - sess.Find(&issueList) resultMap := make(map[int64]int) - log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) - for _, issueRecord := range issueList { - if _, ok := resultMap[issueRecord.PosterID]; !ok { - resultMap[issueRecord.PosterID] = 1 - } else { - resultMap[issueRecord.PosterID] += 1 + cond := "pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueList := make([]*Issue, 0) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&issueList) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -677,34 +765,68 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + + cond := "user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + cond := "op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query Action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -714,16 +836,30 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - commentList := make([]*Comment, 0) - sess.Find(&commentList) + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Comment size=" + fmt.Sprint(len(commentList))) - for _, commentRecord := range commentList { - if _, ok := resultMap[commentRecord.PosterID]; !ok { - resultMap[commentRecord.PosterID] = 1 - } else { - resultMap[commentRecord.PosterID] += 1 + count, err := sess.Where(cond).Count(new(Comment)) + if err != nil { + log.Info("query Comment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,type,poster_id").Table("comment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -733,18 +869,37 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - watchList := make([]*Watch, 0) - sess.Find(&watchList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + resultMap := make(map[int64]int) - log.Info("query Watch size=" + fmt.Sprint(len(watchList))) - for _, watchRecord := range watchList { - if _, ok := resultMap[watchRecord.UserID]; !ok { - resultMap[watchRecord.UserID] = 1 - } else { - resultMap[watchRecord.UserID] += 1 + count, err := sess.Where(cond).Count(new(Watch)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + watchList := make([]*Watch, 0) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&watchList) + + log.Info("query Watch size=" + fmt.Sprint(len(watchList))) + for _, watchRecord := range watchList { + if _, ok := resultMap[watchRecord.UserID]; !ok { + resultMap[watchRecord.UserID] = 1 + } else { + resultMap[watchRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } @@ -753,82 +908,151 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - starList := make([]*Star, 0) - sess.Find(&starList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Star size=" + fmt.Sprint(len(starList))) - for _, starRecord := range starList { - if _, ok := resultMap[starRecord.UID]; !ok { - resultMap[starRecord.UID] = 1 - } else { - resultMap[starRecord.UID] += 1 + + count, err := sess.Where(cond).Count(new(Star)) + if err != nil { + log.Info("query star error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uid,repo_id").Table("star").Where(cond).Limit(Page_SIZE, int(indexTotal)) + starList := make([]*Star, 0) + sess.Find(&starList) + + log.Info("query Star size=" + fmt.Sprint(len(starList))) + for _, starRecord := range starList { + if _, ok := resultMap[starRecord.UID]; !ok { + resultMap[starRecord.UID] = 1 + } else { + resultMap[starRecord.UID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap - } func queryFollow(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - followList := make([]*Follow, 0) - sess.Find(&followList) resultMap := make(map[int64]int) - log.Info("query Follow size=" + fmt.Sprint(len(followList))) - for _, followRecord := range followList { - if _, ok := resultMap[followRecord.FollowID]; !ok { - resultMap[followRecord.FollowID] = 1 - } else { - resultMap[followRecord.FollowID] += 1 + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Follow)) + if err != nil { + log.Info("query follow error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).Limit(Page_SIZE, int(indexTotal)) + followList := make([]*Follow, 0) + sess.Find(&followList) + + log.Info("query Follow size=" + fmt.Sprint(len(followList))) + for _, followRecord := range followList { + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 + } else { + resultMap[followRecord.FollowID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - attachmentList := make([]*Attachment, 0) - sess.Find(&attachmentList) resultMap := make(map[int64]int) - log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) - for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB - } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Attachment)) + if err != nil { + log.Info("query attachment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if _, ok := resultMap[attachRecord.UploaderID]; !ok { + resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + } else { + resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoList := make([]*Repository, 0) - sess.Find(&repoList) resultMap := make(map[int64]int) - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := resultMap[repoRecord.OwnerID]; !ok { - resultMap[repoRecord.OwnerID] = 1 - } else { - resultMap[repoRecord.OwnerID] += 1 + + cond := "is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Repository)) + if err != nil { + log.Info("query Repository error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,owner_id,name").Table("repository").Where(cond).Limit(Page_SIZE, int(indexTotal)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + + statictisSess.Select("id,repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)).OrderBy("id desc") repoStatisticList := make([]*RepoStatistic, 0) statictisSess.Find(&repoStatisticList) repoOpenIIndexMap := make(map[int64]float64) @@ -884,18 +1108,34 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - userLoginLogList := make([]*UserLoginLog, 0) - statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) - log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) - for _, loginRecord := range userLoginLogList { - if _, ok := resultMap[loginRecord.UId]; !ok { - resultMap[loginRecord.UId] = 1 - } else { - resultMap[loginRecord.UId] += 1 + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := statictisSess.Where(cond).Count(new(UserLoginLog)) + if err != nil { + log.Info("query UserLoginLog error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 299133290..0f1c700d2 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -2,6 +2,7 @@ package cloudbrain import ( "errors" + "strconv" "code.gitea.io/gitea/modules/setting" @@ -30,29 +31,30 @@ var ( ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + if !ctx.IsSigned { + return false + } + log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) + log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) if err != nil { + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() } else { + log.Info("is job creator:" + strconv.FormatBool(ctx.User.ID == job.UserID)) return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID } } -func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { - - if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { - return false - } - return isAdminOrOwnerOrJobCreater(ctx, job, nil) -} - -func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { +func CanDeleteJob(ctx *context.Context, job *models.Cloudbrain) bool { return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func CanCreateOrDebugJob(ctx *context.Context) bool { + if !ctx.IsSigned { + return false + } return ctx.Repo.CanWrite(models.UnitTypeCloudBrain) } @@ -62,7 +64,9 @@ func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { } func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + if !ctx.IsSigned { + return false + } if err != nil { return ctx.IsUserSiteAdmin() } else { @@ -113,7 +117,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, log.Error("no such resourceSpecId(%d)", resourceSpecId, ctx.Data["MsgID"]) return errors.New("no such resourceSpec") } - + jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -198,8 +202,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, + Type: models.TypeCloudBrainOne, + Uuid: uuid, }) if err != nil { diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 294690d45..b9838e66f 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -134,7 +134,7 @@ func registerHandleBlockChainUnSuccessRepos() { RegisterTaskFatal("handle_blockchain_unsuccess_repos", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessRepos() return nil @@ -145,7 +145,7 @@ func registerHandleBlockChainMergedPulls() { RegisterTaskFatal("handle_blockchain_merged_pull", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainMergedPulls() return nil @@ -156,7 +156,7 @@ func registerHandleBlockChainUnSuccessCommits() { RegisterTaskFatal("handle_blockchain_unsuccess_commits", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 3m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessCommits() return nil diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0ab1bbc88..2d70e47b1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,6 +433,9 @@ var ( AuthUser string AuthPassword string + //home page + RecommentRepoAddr string + //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -448,21 +451,25 @@ var ( GpuTypes string DebugServerHost string ResourceSpecs string + MaxDuration int64 //benchmark config IsBenchmarkEnabled bool - BenchmarkCode string + BenchmarkOwner string + BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetCode string + Snn4imagenetOwner string + Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreCode string + BrainScoreOwner string + BrainScoreName string BrainScoreServerHost string //blockchain config @@ -1225,6 +1232,9 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") + sec = Cfg.Section("homepage") + RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") @@ -1235,21 +1245,25 @@ func NewContext() { JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug") GpuTypes = sec.Key("GPU_TYPES").MustString("") ResourceSpecs = sec.Key("RESOURCE_SPECS").MustString("") + MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkOwner = sec.Key("OWNER").MustString("") + BenchmarkName = sec.Key("NAME").MustString("") BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetOwner = sec.Key("OWNER").MustString("") + Snn4imagenetName = sec.Key("NAME").MustString("") Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreOwner = sec.Key("OWNER").MustString("") + BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3ab3a00c9..48a010b73 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -778,6 +778,8 @@ datasets = Datasets datasets.desc = Enable Dataset cloudbrain_helper=Use GPU/NPU resources to open notebooks, model training tasks, etc. +model_manager = Model + debug=Debug stop=Stop delete=Delete diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8bd04459a..b1c339080 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -782,7 +782,7 @@ datasets=数据集 datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 -model_manager = 模型管理 +model_manager = 模型 model_noright=无权限操作 debug=调试 @@ -857,7 +857,7 @@ modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 modelarts.train_job.parameter_setting_info=参数信息 modelarts.train_job.fast_parameter_setting=一键式参数配置 -modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 +modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 modelarts.train_job.fast_parameter_setting_config_link=这里 modelarts.train_job.frames=常用框架 modelarts.train_job.algorithm_origin=算法来源 @@ -1987,7 +1987,7 @@ team_unit_desc=允许访问项目单元 team_unit_disabled=(已禁用) form.name_reserved=组织名称 '%s' 是被保留的。 -form.name_pattern_not_allowed=项目名称中不允许使用 "%s"。 +form.name_pattern_not_allowed=组织名称中不允许使用 "%s"。 form.create_org_not_allowed=此账号禁止创建组织 settings=组织设置 diff --git a/public/img/member.svg b/public/img/member.svg new file mode 100644 index 000000000..5e91d1c67 --- /dev/null +++ b/public/img/member.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/pro_num.svg b/public/img/pro_num.svg new file mode 100644 index 000000000..f6d5ae7d6 --- /dev/null +++ b/public/img/pro_num.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/home.go b/routers/home.go index d2c93c771..7cc353ed8 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,8 @@ package routers import ( "bytes" + "fmt" + "io/ioutil" "net/http" "strings" @@ -511,3 +513,43 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } +func RecommendOrgFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "organizations" + recommendFromPromote(ctx, url) +} + +func recommendFromPromote(ctx *context.Context, url string) { + resp, err := http.Get(url) + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + + allLineStr := string(bytes) + lines := strings.Split(allLineStr, "\n") + result := make([]string, len(lines)) + for i, line := range lines { + + tmpIndex := strings.Index(line, ".") + log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) + if tmpIndex == -1 { + result[i] = strings.Trim(line, " ") + } else { + result[i] = strings.Trim(line[tmpIndex+1:], " ") + } + } + ctx.JSON(http.StatusOK, result) +} + +func RecommendRepoFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "projects" + recommendFromPromote(ctx, url) +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index c6ec6c6ae..5103c2a5e 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -28,7 +28,6 @@ const ( func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) - //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { log.Info("query task error." + err.Error()) return err @@ -56,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType = aiTask.Type //download model zip //train type if cloudType == models.TypeCloudBrainTwo { - modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl) if err != nil { log.Info("download model from CloudBrainTwo faild." + err.Error()) return err @@ -71,7 +70,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("accuracyJson=" + string(accuracyJson)) aiTaskJson, _ := json.Marshal(aiTask) - //taskConfigInfo,err := models.GetCloudbrainByJobIDAndVersionName(jobId,aiTask.VersionName) model := &models.AiModelManage{ ID: id, Version: version, @@ -86,7 +84,6 @@ func saveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: ctx.User.ID, - UserName: ctx.User.Name, UserRelAvatarLink: ctx.User.RelAvatarLink(), CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, @@ -144,9 +141,12 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } -func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + if trainUrl != "" { + objectkey = strings.Trim(trainUrl[len(setting.Bucket)+1:], "/") + } + modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { @@ -395,15 +395,42 @@ func ShowSingleModel(ctx *context.Context) { ctx.JSON(http.StatusOK, models) } +func queryUserName(intSlice []int64) map[int64]string { + keys := make(map[int64]string) + uniqueElements := []int64{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = "" + uniqueElements = append(uniqueElements, entry) + } + } + userNames, err := models.GetUserNamesByIDs(uniqueElements) + if err == nil { + for i, userName := range userNames { + keys[uniqueElements[i]] = userName + } + } + return keys +} + func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") aimodels := models.QueryModelByName(name, repoId) - for _, model := range aimodels { + + userIds := make([]int64, len(aimodels)) + for i, model := range aimodels { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + userNameMap := queryUserName(userIds) + + for _, model := range aimodels { + model.UserName = userNameMap[model.UserId] } + if len(aimodels) > 0 { ctx.JSON(200, aimodels[1:]) } else { @@ -431,7 +458,7 @@ func isOper(ctx *context.Context, modelUserId int64) bool { if ctx.User == nil { return false } - if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { + if ctx.User.IsAdmin || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { return true } return false @@ -463,10 +490,17 @@ func ShowModelPageInfo(ctx *context.Context) { return } - for _, model := range modelResult { + userIds := make([]int64, len(modelResult)) + for i, model := range modelResult { log.Info("model=" + model.Name) log.Info("model.UserId=" + fmt.Sprint(model.UserId)) model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNameMap := queryUserName(userIds) + for _, model := range modelResult { + model.UserName = userNameMap[model.UserId] } mapInterface := make(map[string]interface{}) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 07a3a9e8d..256bc9e6d 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "os/exec" "regexp" "sort" "strconv" @@ -71,15 +70,9 @@ func CloudBrainIndex(ctx *context.Context) { return } - timestamp := time.Now().Unix() for i, task := range ciTasks { - if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) { - ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) - } else { - ciTasks[i].CanDebug = false - } - - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } @@ -251,13 +244,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadCode(repo, codePath) uploadCodeToMinio(codePath + "/", jobName, "/code/") - modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath - err = os.MkdirAll(modelPath, os.ModePerm) - if err != nil { - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) - return - } + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" + mkModelPath(modelPath) + uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath + "/") benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { @@ -267,23 +256,25 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuType = gpuInfo.Value } } - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType) uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, getMinioPath(jobName, cloudbrain.ModelMountPath + "/"), + getMinioPath(jobName, cloudbrain.BenchMarkMountPath + "/"), getMinioPath(jobName, cloudbrain.Snn4imagenetMountPath + "/"), + getMinioPath(jobName, cloudbrain.BrainScoreMountPath + "/"), jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) @@ -607,7 +598,7 @@ func getImages(ctx *context.Context, imageType string) { func GetModelDirs(jobName string, parentDir string) (string, error) { var req string - modelActualPath := setting.JobPath + jobName + "/model/" + modelActualPath := getMinioPath(jobName, cloudbrain.ModelMountPath + "/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { @@ -617,6 +608,10 @@ func GetModelDirs(jobName string, parentDir string) (string, error) { return getDirs(req) } +func getMinioPath(jobName, suffixPath string) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + jobName + suffixPath +} + func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -698,19 +693,21 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { +func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } - command := "git clone " + gitPath + " " + codePath - cmd := exec.Command("/bin/bash", "-c", command) - _, err = cmd.Output() - + repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { - log.Error("exec.Command(%s) failed:%v", command, err) + log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) + return err + } + + if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } @@ -772,6 +769,32 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { return nil } +func mkModelPath(modelPath string) error { + err := os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + log.Error("MkdirAll(%s) failed:%v", modelPath, err) + return err + } + + fileName := modelPath + "README" + f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + log.Error("OpenFile failed", err.Error()) + return err + } + + defer f.Close() + + _, err = f.WriteString("You can put the model file into this directory and download it by the web page.") + if err != nil { + log.Error("WriteString failed", err.Error()) + return err + } + + return nil +} + + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { @@ -793,10 +816,24 @@ func SyncCloudbrainStatus() { taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) task.Status = taskRes.TaskStatuses[0].State if task.Status != string(models.JobWaiting) { + task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix() err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) - continue + } + + if task.Duration >= setting.MaxDuration { + log.Info("begin to stop job(%s), because of the duration", task.JobName) + err = cloudbrain.StopJob(task.JobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err) + continue + } + task.Status = string(models.JobStopped) + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } } } } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f0b807d96..3994ba542 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -69,12 +69,9 @@ func NotebookIndex(ctx *context.Context) { } for i, task := range ciTasks { - if task.Status == string(models.JobRunning) { - ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) - } else { - ciTasks[i].CanDebug = false - } - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -306,7 +303,7 @@ func TrainJobIndex(ctx *context.Context) { } for i, task := range tasks { - tasks[i].CanDel = cloudbrain.CanDeleteTrainJob(ctx, &task.Cloudbrain) + tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) } diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 437521d5a..a182e9087 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -532,6 +532,7 @@ func Download(ctx *context.Context) { } ctx.Repo.Repository.IncreaseCloneCnt() + ctx.Repo.Repository.IncreaseGitCloneCnt() ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2f37dc15e..42189c57f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -148,11 +148,6 @@ func QueryUserStaticDataPage(ctx *context.Context) { func TimingCountDataByDateAndReCount(date string, isReCount bool) { - if date == "refreshAll" { - models.RefreshUserStaticAllTabel() - return - } - t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -205,10 +200,9 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } - - if isReCount { - models.RefreshUserStaticAllTabel() - } + log.Info("start to count all user info data") + //models.RefreshUserStaticAllTabel(wikiMap) + log.Info("end to count all user info data") } func TimingCountDataByDate(date string) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a8f820dba..a18fe422c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) + m.Get("/recommend/org", routers.RecommendOrgFromPromote) + m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") diff --git a/routers/user/profile.go b/routers/user/profile.go index fe5dc4598..9b99fa741 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -214,10 +214,15 @@ func Profile(ctx *context.Context) { total = int(count) case "datasets": + var isOwner = false + if ctx.User != nil && ctx.User.ID == ctxUser.ID { + isOwner = true + } datasetSearchOptions := &models.SearchDatasetOptions{ Keyword: keyword, OwnerID: ctxUser.ID, SearchOrderBy: orderBy, + IsOwner: isOwner, ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.ExplorePagingNum, diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index c87524afb..27c82a6bf 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,7 +17,7 @@ {{if .IsSigned}} -
+
{{.i18n.Tr "index"}}   @@ -47,7 +47,7 @@
{{else if .IsLandingPageHome}} -
+
{{.i18n.Tr "home"}}   @@ -94,13 +94,13 @@ */}} {{if .IsSigned}} -
+
- + @@ -190,12 +190,13 @@
+
- + diff --git a/templates/base/head_navbar_fluid.tmpl b/templates/base/head_navbar_fluid.tmpl index d323d2151..5cd889c51 100644 --- a/templates/base/head_navbar_fluid.tmpl +++ b/templates/base/head_navbar_fluid.tmpl @@ -17,7 +17,7 @@ {{if .IsSigned}} -
+
{{.i18n.Tr "index"}}   @@ -46,7 +46,7 @@
{{else if .IsLandingPageHome}} -
+
{{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_home.tmpl b/templates/base/head_navbar_home.tmpl index 096722bf6..7391fa3e5 100644 --- a/templates/base/head_navbar_home.tmpl +++ b/templates/base/head_navbar_home.tmpl @@ -9,7 +9,7 @@
{{if .IsSigned}} -
+
{{.i18n.Tr "index"}}   @@ -38,7 +38,7 @@
{{else if .IsLandingPageHome}} -
+
{{.i18n.Tr "home"}}   diff --git a/templates/base/head_navbar_pro.tmpl b/templates/base/head_navbar_pro.tmpl new file mode 100644 index 000000000..086e4de02 --- /dev/null +++ b/templates/base/head_navbar_pro.tmpl @@ -0,0 +1,216 @@ + diff --git a/templates/base/head_pro.tmpl b/templates/base/head_pro.tmpl new file mode 100644 index 000000000..0b92016d1 --- /dev/null +++ b/templates/base/head_pro.tmpl @@ -0,0 +1,208 @@ + + + + + + + {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} + + {{if UseServiceWorker}} + + {{else}} + + {{end}} + + + + + + + {{if .IsSigned}} + + {{end}} + {{if .ContextUser}} + + {{end}} + {{if .SearchLimit}} + + {{end}} +{{if .GoGetImport}} + + +{{end}} + + + + + + + + +{{if .RequireSimpleMDE}} + +{{end}} + +{{if .RequireTribute}} + +{{end}} + + + +{{if .RequireMinicolors}} + +{{end}} + +{{if .PageIsUserProfile}} + + + + + {{if .Owner.Description}} + + {{end}} +{{else if .Repository}} + {{if .Issue}} + + + {{if .Issue.Content}} + + {{end}} + {{else}} + + + {{if .Repository.Description}} + + {{end}} + {{end}} + + +{{else}} + + + + + +{{end}} + +{{if .IsSigned }} + {{ if ne .SignedUser.Theme "gitea" }} + + {{end}} +{{else if ne DefaultTheme "gitea"}} + +{{end}} +{{template "custom/header" .}} + + + + + + + {{template "custom/body_outer_pre" .}} + +
+ + + {{template "custom/body_inner_pre" .}} + + {{if not .PageIsInstall}} +
+ {{template "base/head_navbar_pro" .}} +
+ {{end}} +{{/* +
+ + +*/}} diff --git a/templates/explore/repos.tmpl b/templates/explore/repos.tmpl index f487c9e01..df305099a 100644 --- a/templates/explore/repos.tmpl +++ b/templates/explore/repos.tmpl @@ -1,4 +1,4 @@ -{{template "base/head" .}} +{{template "base/head_pro" .}}
{{template "explore/repo_search" .}} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index b84f4771f..f5376b07c 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -134,7 +134,8 @@ {{if .Permission.CanRead $.UnitTypeDatasets}} - {{svg "octicon-inbox" 16}} {{.i18n.Tr "datasets"}} + + {{.i18n.Tr "datasets"}} {{end}} {{if .Permission.CanRead $.UnitTypeModelManage}} @@ -145,7 +146,11 @@ {{end}} {{if .Permission.CanRead $.UnitTypeCloudBrain}} - {{svg "octicon-server" 16}} {{.i18n.Tr "repo.cloudbrain"}} + + + {{.i18n.Tr "repo.cloudbrain"}} + + {{end}} + + {{range .Orgs}} +
    + + {{if (or .Visibility.IsPublic (and ($.SignedUser) (or .Visibility.IsLimited (and (.IsUserPartOfOrg $.SignedUserID) .Visibility.IsPrivate) ($.IsAdmin))))}} +
  • + +
    • +
  • + {{.Name}} +
    • + +
  • + + + {{.NumMembers}} +
    • +
  • + + {{.NumRepos}} +
    • + {{end}} + +
+ {{end}} {{end}} {{if and .IsSigned (ne .SignedUserName .Owner.Name)}} @@ -79,13 +109,6 @@ {{end}}
-
-
    -
  • -
        -
        • -
      -
      @@ -155,3 +178,22 @@
      {{template "base/footer" .}} + + \ No newline at end of file diff --git a/web_src/less/openi.less b/web_src/less/openi.less index 0f3129cd5..a20563ea5 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -518,4 +518,10 @@ display: block; } .letf2{ margin-left: -2px; -} \ No newline at end of file +} +.edge{ + margin-left:0 !important; + margin-right: 0 !important; + padding-left:0 !important; + padding-right:0 !important; + } \ No newline at end of file