diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a7d549cd4..8b7691677 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -71,6 +71,49 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` } +func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { + log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + statictisSess.Find(&userBusinessAnalysisList) + + resultMap := make(map[int64]*UserBusinessAnalysis) + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) + index := 0 + for _, v := range resultMap { + userBusinessAnalysisReturnList[index] = v + index += 1 + } + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList +} + func CountData(wikiCountMap map[string]int) { log.Info("start to count other user info data") sess := x.NewSession() @@ -92,7 +135,7 @@ func CountData(wikiCountMap map[string]int) { CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) - CodeMergeCountMap := queryAction(start_unix, end_unix, 11) + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 10) @@ -223,6 +266,28 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { } +func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + + sess.Select("issue.*").Table("issue"). + Join("inner", "pull_request", "issue.id=pull_request.issue_id"). + Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) + + issueList := make([]*Issue, 0) + sess.Find(&issueList) + resultMap := make(map[int64]int) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + return resultMap +} + func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() @@ -341,7 +406,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) repoList := make([]*Repository, 0) sess.Find(&repoList) resultMap := make(map[int64]int) @@ -354,7 +419,6 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { } } return resultMap - } func subMonth(t1, t2 time.Time) (month int) { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 44eaa8bc1..f9674ddc2 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -495,6 +495,9 @@ var ( Index string TimeField string ElkTimeFormat string + + //nginx proxy + PROXYURL string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1206,6 +1209,7 @@ func NewContext() { Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/") + PROXYURL = sec.Key("PROXY_URL").MustString("") sec = Cfg.Section("modelarts") ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") @@ -1227,6 +1231,7 @@ func NewContext() { Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") + } func loadInternalToken(sec *ini.Section) string { diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 77fe49b4e..d5817fd3f 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,11 +5,14 @@ package storage import ( - "github.com/unknwon/com" + "fmt" + "io" "path" "strconv" "strings" + "github.com/unknwon/com" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" @@ -102,6 +105,49 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { return nil } +func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { + input := &obs.UploadPartInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.UploadId = uploadId + input.PartNumber = partNumber + input.Body = putBody + output, err := ObsCli.UploadPart(input) + if err == nil { + log.Info("RequestId:%s\n", output.RequestId) + log.Info("ETag:%s\n", output.ETag) + return nil + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info(obsError.Code) + log.Info(obsError.Message) + return obsError + } else { + log.Error("error:", err.Error()) + return err + } + } + +} + +func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { + input := &obs.GetObjectInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + output, err := ObsCli.GetObject(input) + if err == nil { + log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", + output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) + return output.Body, nil + } else if obsError, ok := err.(obs.ObsError); ok { + fmt.Printf("Code:%s\n", obsError.Code) + fmt.Printf("Message:%s\n", obsError.Message) + return nil, obsError + } else { + return nil, err + } +} + func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} diff --git a/public/self/labelTaskPage.js b/public/self/labelTaskPage.js index a160c3961..03acd6df1 100644 --- a/public/self/labelTaskPage.js +++ b/public/self/labelTaskPage.js @@ -309,11 +309,11 @@ function label_task_create(task_name, relate_task_id, taskType,assign_user_id,la success:function(res){ console.log(res); if(res.code == 0){ - alert("自动标注任务创建成功!"); + alert("标注任务创建成功!"); createsucced = true; } else{ - alert("创建自动标注任务失败，" + res.message); + alert("创建标注任务失败，" + res.message); createsucced = false; } }, diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index a8b2c8fbe..59f297a49 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -262,10 +262,15 @@ func GetAttachment(ctx *context.Context) { return } } else { - url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) - if err != nil { - ctx.ServerError("ObsGetPreSignedUrl", err) - return + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_download?uuid=" + attach.UUID + "&file_name=" + attach.Name + log.Info("return url=" + url) + } else { + url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) + if err != nil { + ctx.ServerError("ObsGetPreSignedUrl", err) + return + } } } @@ -273,7 +278,6 @@ func GetAttachment(ctx *context.Context) { ctx.ServerError("Update", err) return } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } else { fr, err := storage.Attachments.Open(attach.RelativePath()) @@ -282,7 +286,6 @@ func GetAttachment(ctx *context.Context) { return } defer fr.Close() - if err = increaseDownloadCount(attach, dataSet); err != nil { ctx.ServerError("Update", err) return @@ -662,6 +665,53 @@ func NewMultipart(ctx *context.Context) { } } +func PutOBSProxyUpload(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Query("uploadId") + partNumber := ctx.QueryInt("partNumber") + fileName := ctx.Query("file_name") + + RequestBody := ctx.Req.Body() + + if RequestBody == nil { + ctx.Error(500, fmt.Sprintf("FormFile: %v", RequestBody)) + return + } + + err := storage.ObsMultiPartUpload(uuid, uploadID, partNumber, fileName, RequestBody.ReadCloser()) + if err != nil { + log.Info("upload error.") + } +} + +func GetOBSProxyDownload(ctx *context.Context) { + uuid := ctx.Query("uuid") + fileName := ctx.Query("file_name") + + body, err := storage.ObsDownload(uuid, fileName) + if err != nil { + log.Info("upload error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } +} + func GetMultipartUploadUrl(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") @@ -689,10 +739,16 @@ func GetMultipartUploadUrl(ctx *context.Context) { return } } else { - url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) - if err != nil { - ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) - return + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName + log.Info("return url=" + url) + } else { + url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) + if err != nil { + ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) + return + } + log.Info("url=" + url) } } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 127c034ca..546268a82 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,13 +6,15 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/operation" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/operation" + "code.gitea.io/gitea/routers/private" + "code.gitea.io/gitea/routers/secure" "code.gitea.io/gitea/models" @@ -33,7 +35,6 @@ import ( "code.gitea.io/gitea/routers/dev" "code.gitea.io/gitea/routers/events" "code.gitea.io/gitea/routers/org" - "code.gitea.io/gitea/routers/private" "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/routers/user" userSetting "code.gitea.io/gitea/routers/user/setting" @@ -573,6 +574,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/private", repo.UpdatePublicAttachment) m.Get("/get_chunks", repo.GetSuccessChunks) m.Get("/new_multipart", repo.NewMultipart) + m.Put("/obs_proxy_multipart", repo.PutOBSProxyUpload) + m.Get("/obs_proxy_download", repo.GetOBSProxyDownload) m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) m.Post("/complete_multipart", repo.CompleteMultipart) m.Post("/update_chunk", repo.UpdateMultipart) diff --git a/templates/repo/datasets/label/index.tmpl b/templates/repo/datasets/label/index.tmpl index cb268c5b9..9a0863974 100644 --- a/templates/repo/datasets/label/index.tmpl +++ b/templates/repo/datasets/label/index.tmpl @@ -123,7 +123,7 @@