From 18c60d38ab55f161b2d14a2094ceb682ce1f85ff Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Oct 2021 17:01:43 +0800 Subject: [PATCH 001/206] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/serv.go | 2 +- routers/repo/http.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/private/serv.go b/routers/private/serv.go index f405d1d75..c2c6b656c 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -125,7 +125,7 @@ func ServCommand(ctx *macaron.Context) { return } } - + log.Info("git oper to this.") for _, verb := range ctx.QueryStrings("verb") { // clone_cnt if verb == "git-upload-pack" { go repo.IncreaseCloneCnt() diff --git a/routers/repo/http.go b/routers/repo/http.go index ed6276466..7a1670402 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -312,7 +312,7 @@ func HTTP(ctx *context.Context) { } environ = append(environ, models.ProtectedBranchRepoID+fmt.Sprintf("=%d", repo.ID)) - + log.Info("git oper to this 2.") if service == "git-upload-pack" { // clone_cnt go repo.IncreaseCloneCnt() } From cd9abbbf19c2f32131d52d7f150b1cc705b3cdc8 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Oct 2021 17:17:59 +0800 Subject: [PATCH 002/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/http.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/routers/repo/http.go b/routers/repo/http.go index 7a1670402..7199cd42b 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -9,6 +9,7 @@ import ( "bytes" "compress/gzip" gocontext "context" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -313,6 +314,13 @@ func HTTP(ctx *context.Context) { environ = append(environ, models.ProtectedBranchRepoID+fmt.Sprintf("=%d", repo.ID)) log.Info("git oper to this 2.") + + dataJson, _ := json.Marshal(ctx.Data) + log.Info("dataJson=" + string(dataJson)) + + reqJson, _ := json.Marshal(ctx.Req.Request) + log.Info("reqJson=" + string(reqJson)) + if service == "git-upload-pack" { // clone_cnt go repo.IncreaseCloneCnt() } From 1efd08a43d33517e906a39474e57862f1d87a3c4 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Oct 2021 17:20:09 +0800 Subject: [PATCH 003/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/http.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routers/repo/http.go b/routers/repo/http.go index 7199cd42b..707cb2700 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -318,9 +318,12 @@ func HTTP(ctx *context.Context) { dataJson, _ := json.Marshal(ctx.Data) log.Info("dataJson=" + string(dataJson)) - reqJson, _ := json.Marshal(ctx.Req.Request) + reqJson, _ := json.Marshal(ctx.Req) log.Info("reqJson=" + string(reqJson)) + paramJson, _ := json.Marshal(ctx.Params) + log.Info("paramJson=" + string(paramJson)) + if service == "git-upload-pack" { // clone_cnt go repo.IncreaseCloneCnt() } From e01a7a26547b3ecf4c204299d775c3221f1cad2d Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 29 Oct 2021 17:23:51 +0800 Subject: [PATCH 004/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/http.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/routers/repo/http.go b/routers/repo/http.go index 707cb2700..172cf9663 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -318,9 +318,13 @@ func HTTP(ctx *context.Context) { dataJson, _ := json.Marshal(ctx.Data) log.Info("dataJson=" + string(dataJson)) - reqJson, _ := json.Marshal(ctx.Req) + reqJson, _ := json.Marshal(ctx.Req.Form) log.Info("reqJson=" + string(reqJson)) + log.Info("query go-get:" + ctx.Query("go-get")) + + log.Info("query service:" + ctx.Query("service")) + paramJson, _ := json.Marshal(ctx.Params) log.Info("paramJson=" + string(paramJson)) From 69858de7d0bd483ed2662a2348fa38372df84f86 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 2 Nov 2021 10:23:22 +0800 Subject: [PATCH 005/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 26 ++++++++++++++++++++++++++ models/models.go | 1 + models/repo.go | 23 ++++++++++++++++++++++- routers/private/internal.go | 1 - routers/repo/http.go | 6 ++++++ routers/repo/user_data_analysis.go | 14 ++++++++++++++ routers/routes/routes.go | 2 +- 7 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 models/ai_model_manage.go diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go new file mode 100644 index 000000000..93414795b --- /dev/null +++ b/models/ai_model_manage.go @@ -0,0 +1,26 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" +) + +type AiModelManage struct { + ID int64 `xorm:"pk"` + Name string `xorm:"NOT NULL"` + Version string `xorm:"NOT NULL"` + Parent int64 `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + ConfigJson string `xorm:"text"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + DatasetId int64 `xorm:"NULL"` + RepoId int64 `xorm:"NULL"` + CodePath string `xorm:"varchar(400) NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` +} diff --git a/models/models.go b/models/models.go index 696d0949b..32767b544 100755 --- a/models/models.go +++ b/models/models.go @@ -133,6 +133,7 @@ func init() { new(FileChunk), new(BlockChain), new(RecommendOrg), + new(AiModelManage), ) tablesStatistic = append(tablesStatistic, diff --git a/models/repo.go b/models/repo.go index c8629875e..1a5cf122c 100755 --- a/models/repo.go +++ b/models/repo.go @@ -210,9 +210,12 @@ type Repository struct { Balance string `xorm:"NOT NULL DEFAULT '0'"` BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0"` - // git clone total count + // git clone and git pull total count CloneCnt int64 `xorm:"NOT NULL DEFAULT 0"` + // only git clone total count + GitCloneCnt int64 `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` @@ -2473,6 +2476,24 @@ func (repo *Repository) IncreaseCloneCnt() { return } +func (repo *Repository) IncreaseGitCloneCnt() { + sess := x.NewSession() + defer sess.Close() + + if err := sess.Begin(); err != nil { + return + } + if _, err := sess.Exec("UPDATE `repository` SET git_clone_cnt = git_clone_cnt + 1 WHERE id = ?", repo.ID); err != nil { + return + } + + if err := sess.Commit(); err != nil { + return + } + + return +} + func UpdateRepositoryCommitNum(repo *Repository) error { if _, err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil { return err diff --git a/routers/private/internal.go b/routers/private/internal.go index b254d48ba..b029eedf9 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -44,6 +44,5 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat", RepoStatisticManually) - }, CheckInternalToken) } diff --git a/routers/repo/http.go b/routers/repo/http.go index 172cf9663..cab29ba05 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -332,6 +332,12 @@ func HTTP(ctx *context.Context) { go repo.IncreaseCloneCnt() } + if ctx.Req.Method == "POST" { + if strings.HasSuffix(ctx.Req.URL.Path, "git-upload-pack") { + go repo.IncreaseGitCloneCnt() + } + } + w := ctx.Resp r := ctx.Req.Request cfg := &serviceConfig{ diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 7dc7af321..b0ae58e93 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -1,13 +1,27 @@ package repo import ( + "fmt" + "net/http" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "gopkg.in/macaron.v1" ) +func QueryUserStaticData(ctx *macaron.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + log.Info("startDate=" + startDate + " endDate=" + endDate) + startTime, _ := time.Parse("2006-01-02", startDate) + endTime, _ := time.Parse("2006-01-02", endDate) + log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) + +} + func TimingCountDataByDate(date string) { t, _ := time.Parse("2006-01-02", date) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 7e7d0642a..7819e3930 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -786,7 +786,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - + m.Get("/tool/query_user_static", repo.QueryUserStaticData) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Group("/milestone", func() { From 3b2c6c631614b310069f128f34e03739e5b9c7fb Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 2 Nov 2021 17:27:15 +0800 Subject: [PATCH 006/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 17 +++++++++++- routers/repo/ai_model_manage.go | 53 ++++++++++++++++++++++++++++++++++++++ routers/repo/user_data_analysis.go | 4 +-- 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 routers/repo/ai_model_manage.go diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 93414795b..60c45fe8c 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -1,11 +1,14 @@ package models import ( + "fmt" + + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) type AiModelManage struct { - ID int64 `xorm:"pk"` + ID string `xorm:"pk"` Name string `xorm:"NOT NULL"` Version string `xorm:"NOT NULL"` Parent int64 `xorm:"NOT NULL"` @@ -24,3 +27,15 @@ type AiModelManage struct { CodePath string `xorm:"varchar(400) NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` } + +func SaveModelToDb(model *AiModelManage) error { + sess := x.NewSession() + defer sess.Close() + + re, err := sess.Insert(model) + if err != nil { + return err + } + log.Info("success to save db.re=" + fmt.Sprint((re))) + return nil +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go new file mode 100644 index 000000000..98455e92f --- /dev/null +++ b/routers/repo/ai_model_manage.go @@ -0,0 +1,53 @@ +package repo + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + uuid "github.com/satori/go.uuid" +) + +func SaveModel(ctx *context.Context) { + log.Info("save model start.") + jobId := ctx.QueryInt64("JobId") + name := ctx.Query("Name") + + aiTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + JobID: jobId, + }) + if err != nil { + log.Info("query task error.") + return + } + if count > 0 { + for _, task := range aiTasks { + log.Info("find task name:" + task.JobName) + + } + } + + id := uuid.NewV4() + + model := &models.AiModelManage{ + ID: id.String(), + Name: name, + } + + models.SaveModelToDb(model) + + log.Info("save model end.") +} + +func DeleteModel(ctx *context.Context) { + log.Info("delete model start.") +} + +func DownloadModel(ctx *context.Context) { + log.Info("download model start.") + +} + +func ShowModelInfo(ctx *context.Context) { + log.Info("ShowModelInfo.") + +} diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index b0ae58e93..68cccd478 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -6,12 +6,12 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - "gopkg.in/macaron.v1" ) -func QueryUserStaticData(ctx *macaron.Context) { +func QueryUserStaticData(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") log.Info("startDate=" + startDate + " endDate=" + endDate) From f9f9286d09b2ced5faf03dff6bedfe1b9189e55d Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 09:56:16 +0800 Subject: [PATCH 007/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 35 ++++++++- routers/repo/ai_model_manage.go | 161 +++++++++++++++++++++++++++++++++++++--- routers/repo/cloudbrain.go | 12 +-- 3 files changed, 189 insertions(+), 19 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 60c45fe8c..b6169b806 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -11,7 +11,7 @@ type AiModelManage struct { ID string `xorm:"pk"` Name string `xorm:"NOT NULL"` Version string `xorm:"NOT NULL"` - Parent int64 `xorm:"NOT NULL"` + Parent string `xorm:"NOT NULL"` Type int `xorm:"NOT NULL"` Size int64 `xorm:"NOT NULL"` Description string `xorm:"varchar(2000)"` @@ -22,10 +22,14 @@ type AiModelManage struct { Engine int `xorm:"NOT NULL DEFAULT 0"` Status int `xorm:"NOT NULL DEFAULT 0"` Accuracy string `xorm:"varchar(1000)"` - DatasetId int64 `xorm:"NULL"` + AttachmentId string `xorm:"NULL"` RepoId int64 `xorm:"NULL"` - CodePath string `xorm:"varchar(400) NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } func SaveModelToDb(model *AiModelManage) error { @@ -39,3 +43,28 @@ func SaveModelToDb(model *AiModelManage) error { log.Info("success to save db.re=" + fmt.Sprint((re))) return nil } + +func DeleteModelById(id string) error { + sess := x.NewSession() + defer sess.Close() + + re, err := sess.Delete(&AiModelManage{ + ID: id, + }) + if err != nil { + return err + } + log.Info("success to delete from db.re=" + fmt.Sprint((re))) + return nil + +} + +func QueryModelByName(name string, uid int64) []*AiModelManage { + sess := x.NewSession() + defer sess.Close() + sess.Select("*").Table("ai_model_manage"). + Where("name=" + name + " and user_id=" + fmt.Sprint(uid)) + aiModelManageList := make([]*AiModelManage, 0) + sess.Find(&aiModelManageList) + return aiModelManageList +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 98455e92f..2e75c5634 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -1,36 +1,92 @@ package repo import ( + "archive/zip" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" uuid "github.com/satori/go.uuid" ) func SaveModel(ctx *context.Context) { log.Info("save model start.") - jobId := ctx.QueryInt64("JobId") + trainTaskId := ctx.QueryInt64("TrainTask") name := ctx.Query("Name") + version := ctx.Query("Version") + label := ctx.Query("Label") + description := ctx.Query("Description") - aiTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ - JobID: jobId, + aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ + JobID: trainTaskId, }) if err != nil { - log.Info("query task error.") + log.Info("query task error." + err.Error()) + ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) return } - if count > 0 { - for _, task := range aiTasks { - log.Info("find task name:" + task.JobName) + uuid := uuid.NewV4() + id := uuid.String() + modelPath := id + parent := id + var modelSize int64 + cloudType := models.TypeCloudBrainTwo + if len(aiTasks) != 1 { + log.Info("query task error. len=" + fmt.Sprint(len(aiTasks))) + ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) + return + } + aiTask := aiTasks[0] + log.Info("find task name:" + aiTask.JobName) + aimodels := models.QueryModelByName(name, ctx.User.ID) + if len(aimodels) > 0 { + for _, model := range aimodels { + if model.ID == model.Parent { + parent = model.ID + } } } + cloudType = aiTask.Cloudbrain.Type + //download model zip + if cloudType == models.TypeCloudBrainOne { + modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "") + if err != nil { + log.Info("download model from CloudBrainOne faild." + err.Error()) + ctx.Error(500, fmt.Sprintf("%v", err)) + return + } + } else if cloudType == models.TypeCloudBrainTwo { + modelPath, err = downloadModelFromCloudBrainTwo(id) + if err == nil { - id := uuid.NewV4() + } else { + log.Info("download model from CloudBrainTwo faild." + err.Error()) + ctx.Error(500, fmt.Sprintf("%v", err)) + return + } + } model := &models.AiModelManage{ - ID: id.String(), - Name: name, + ID: id, + Version: version, + Label: label, + Name: name, + Description: description, + Parent: parent, + Type: cloudType, + Path: modelPath, + Size: modelSize, + AttachmentId: aiTask.Uuid, + RepoId: aiTask.RepoID, + UserId: ctx.User.ID, } models.SaveModelToDb(model) @@ -38,8 +94,93 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } +func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) { + + modelActualPath := setting.Attachment.Minio.RealPath + + setting.Attachment.Minio.Bucket + "/" + + "aimodels/" + + models.AttachmentRelativePath(modelUUID) + + "/" + os.MkdirAll(modelActualPath, 0755) + zipFile := modelActualPath + "model.zip" + + modelDir := setting.JobPath + jobName + "/model/" + + dir, _ := ioutil.ReadDir(modelDir) + if len(dir) == 0 { + return "", 0, errors.New("cannot create model, as model is empty.") + } + + err := zipDir(modelDir, zipFile) + if err != nil { + return "", 0, err + } + + fi, err := os.Stat(zipFile) + if err == nil { + return modelActualPath, fi.Size(), nil + } else { + return "", 0, err + } +} + +func zipDir(dir, zipFile string) error { + fz, err := os.Create(zipFile) + if err != nil { + log.Info("Create zip file failed: %s\n", err.Error()) + return err + } + defer fz.Close() + + w := zip.NewWriter(fz) + defer w.Close() + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + fDest, err := w.Create(path[len(dir)+1:]) + if err != nil { + log.Info("Create failed: %s\n", err.Error()) + return err + } + fSrc, err := os.Open(path) + if err != nil { + log.Info("Open failed: %s\n", err.Error()) + return err + } + defer fSrc.Close() + _, err = io.Copy(fDest, fSrc) + if err != nil { + log.Info("Copy failed: %s\n", err.Error()) + return err + } + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func downloadModelFromCloudBrainTwo(modelUUID string) (string, error) { + dataActualPath := setting.Bucket + "/" + + "aimodels/" + + models.AttachmentRelativePath(modelUUID) + + "/" + return dataActualPath, nil +} + func DeleteModel(ctx *context.Context) { log.Info("delete model start.") + id := ctx.Query("ID") + err := models.DeleteModelById(id) + if err != nil { + ctx.JSON(500, err.Error()) + } else { + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) + } } func DownloadModel(ctx *context.Context) { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 03fba6cd1..b70ec5fda 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -323,7 +323,7 @@ func CloudBrainDebug(ctx *context.Context) { var jobID = ctx.Params(":jobid") if !ctx.IsSigned { log.Error("the user has not signed in") - ctx.Error(http.StatusForbidden, "","the user has not signed in") + ctx.Error(http.StatusForbidden, "", "the user has not signed in") return } task, err := models.GetCloudbrainByJobID(jobID) @@ -340,7 +340,7 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain var jobID = ctx.Params(":jobid") if !ctx.IsSigned { log.Error("the user has not signed in") - ctx.Error(http.StatusForbidden, "","the user has not signed in") + ctx.Error(http.StatusForbidden, "", "the user has not signed in") return } task, err := models.GetCloudbrainByJobID(jobID) @@ -513,10 +513,10 @@ func CloudBrainShowModels(ctx *context.Context) { } //get dirs - dirs, err := getModelDirs(task.JobName, parentDir) + dirs, err := GetModelDirs(task.JobName, parentDir) if err != nil { - log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("getModelDirs failed:", err) + log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetModelDirs failed:", err) return } @@ -567,7 +567,7 @@ func getImages(ctx *context.Context, imageType string) { log.Info("Get images end") } -func getModelDirs(jobName string, parentDir string) (string, error) { +func GetModelDirs(jobName string, parentDir string) (string, error) { var req string modelActualPath := setting.JobPath + jobName + "/model/" if parentDir == "" { From 76ea89deeb02d0a350c8f8a86a2fefaf7db5aaa6 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 15:57:06 +0800 Subject: [PATCH 008/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 31 +++++++++++--- routers/repo/ai_model_manage.go | 94 +++++++++++++++++++++++------------------ routers/repo/modelarts.go | 3 ++ 3 files changed, 81 insertions(+), 47 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index bd73281d0..87e972572 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -174,6 +174,25 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } +func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { + input := &obs.CopyObjectInput{} + input.Bucket = srcBucket + input.Key = srcKeyName + input.CopySourceBucket = destBucket + input.CopySourceKey = destKeyName + _, err := ObsCli.CopyObject(input) + if err == nil { + log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info(obsError.Code) + log.Info(obsError.Message) + } + return err + } + return nil +} + func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket @@ -184,12 +203,12 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool - var fileName,nextParentDir string + var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { fileName = str1[len(str1)-2] isDir = true nextParentDir = fileName - if fileName == parentDir || (fileName + "/") == setting.OutPutPath { + if fileName == parentDir || (fileName+"/") == setting.OutPutPath { continue } } else { @@ -198,10 +217,10 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } fileInfo := FileInfo{ - ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, - Size: val.Size, - IsDir:isDir, + Size: val.Size, + IsDir: isDir, ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) @@ -242,7 +261,7 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") - + input.Expires = 60 * 60 input.Method = obs.HttpMethodGet diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 2e75c5634..1dd1e82b2 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" uuid "github.com/satori/go.uuid" ) @@ -55,16 +56,9 @@ func SaveModel(ctx *context.Context) { } } cloudType = aiTask.Cloudbrain.Type - //download model zip - if cloudType == models.TypeCloudBrainOne { - modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "") - if err != nil { - log.Info("download model from CloudBrainOne faild." + err.Error()) - ctx.Error(500, fmt.Sprintf("%v", err)) - return - } - } else if cloudType == models.TypeCloudBrainTwo { - modelPath, err = downloadModelFromCloudBrainTwo(id) + //download model zip //train type + if cloudType == models.TypeCloudBrainTrainJob { + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") if err == nil { } else { @@ -94,6 +88,55 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { + dataActualPath := setting.Bucket + "/" + + "aimodels/" + + models.AttachmentRelativePath(modelUUID) + + "/" + + models, err := storage.GetObsListObject(jobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + return "", 0, err + } + if len(models) == 0 { + return "", 0, errors.New("cannot create model, as model is empty.") + } + + for _, modelFile := range models { + log.Info("copy file, bucket=%s, src keyname=%s,dest keyname=%s", setting.Bucket, modelFile.ParenDir+modelFile.FileName, dataActualPath+modelFile.FileName) + // err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) + // if err != nil { + // log.Info("copy failed.") + // } + } + + return dataActualPath, 0, nil +} + +func DeleteModel(ctx *context.Context) { + log.Info("delete model start.") + id := ctx.Query("ID") + err := models.DeleteModelById(id) + if err != nil { + ctx.JSON(500, err.Error()) + } else { + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) + } +} + +func DownloadModel(ctx *context.Context) { + log.Info("download model start.") + +} + +func ShowModelInfo(ctx *context.Context) { + log.Info("ShowModelInfo.") + +} + func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) { modelActualPath := setting.Attachment.Minio.RealPath + @@ -161,34 +204,3 @@ func zipDir(dir, zipFile string) error { } return nil } - -func downloadModelFromCloudBrainTwo(modelUUID string) (string, error) { - dataActualPath := setting.Bucket + "/" + - "aimodels/" + - models.AttachmentRelativePath(modelUUID) + - "/" - return dataActualPath, nil -} - -func DeleteModel(ctx *context.Context) { - log.Info("delete model start.") - id := ctx.Query("ID") - err := models.DeleteModelById(id) - if err != nil { - ctx.JSON(500, err.Error()) - } else { - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) - } -} - -func DownloadModel(ctx *context.Context) { - log.Info("download model start.") - -} - -func ShowModelInfo(ctx *context.Context) { - log.Info("ShowModelInfo.") - -} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index ea8ab0c06..ad50c743a 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1107,6 +1107,9 @@ func TrainJobShowModels(ctx *context.Context) { jobID := ctx.Params(":jobid") parentDir := ctx.Query("parentDir") + + log.Info("parentDir=" + parentDir) + dirArray := strings.Split(parentDir, "/") task, err := models.GetCloudbrainByJobID(jobID) if err != nil { From 450993bd46257dc2855c4be37bf97668515051b8 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 16:05:17 +0800 Subject: [PATCH 009/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/tool.go | 12 ++++++++ routers/repo/ai_model_manage.go | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/routers/private/tool.go b/routers/private/tool.go index b93f17090..68de67aa7 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -44,3 +44,15 @@ func RepoStatisticManually(ctx *macaron.Context) { repo.SummaryStatisticDaily(date) repo.TimingCountDataByDate(date) } + +func CreateModel(ctx *macaron.Context) { + trainTaskId := ctx.QueryInt64("TrainTask") + name := ctx.Query("Name") + version := ctx.Query("Version") + label := ctx.Query("Label") + description := ctx.Query("Description") + userId := ctx.QueryInt64("userId") + + repo.SaveModelByParameters(trainTaskId, name, version, label, description, userId) + +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 1dd1e82b2..ec99a8048 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -17,6 +17,70 @@ import ( uuid "github.com/satori/go.uuid" ) +func SaveModelByParameters(trainTaskId int64, name string, version string, label string, description string, userId int64) { + aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ + JobID: trainTaskId, + }) + if err != nil { + log.Info("query task error." + err.Error()) + //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) + return + } + uuid := uuid.NewV4() + id := uuid.String() + modelPath := id + parent := id + var modelSize int64 + cloudType := models.TypeCloudBrainTwo + + if len(aiTasks) != 1 { + log.Info("query task error. len=" + fmt.Sprint(len(aiTasks))) + //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) + return + } + aiTask := aiTasks[0] + log.Info("find task name:" + aiTask.JobName) + aimodels := models.QueryModelByName(name, userId) + if len(aimodels) > 0 { + for _, model := range aimodels { + if model.ID == model.Parent { + parent = model.ID + } + } + } + cloudType = aiTask.Cloudbrain.Type + //download model zip //train type + if cloudType == models.TypeCloudBrainTrainJob { + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") + if err == nil { + + } else { + log.Info("download model from CloudBrainTwo faild." + err.Error()) + //ctx.Error(500, fmt.Sprintf("%v", err)) + return + } + } + + model := &models.AiModelManage{ + ID: id, + Version: version, + Label: label, + Name: name, + Description: description, + Parent: parent, + Type: cloudType, + Path: modelPath, + Size: modelSize, + AttachmentId: aiTask.Uuid, + RepoId: aiTask.RepoID, + UserId: userId, + } + + models.SaveModelToDb(model) + + log.Info("save model end.") +} + func SaveModel(ctx *context.Context) { log.Info("save model start.") trainTaskId := ctx.QueryInt64("TrainTask") From cea33494bb7eb0504c036450de15f7b2d991a0fd Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 16:09:42 +0800 Subject: [PATCH 010/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/internal.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/private/internal.go b/routers/private/internal.go index b029eedf9..5b1d63bff 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -44,5 +44,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat", RepoStatisticManually) + m.Post("/tool/create_model", CreateModel) }, CheckInternalToken) } From ad7790ee1a299e68858f2bc9c7f2bef8ca0876f2 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 16:17:48 +0800 Subject: [PATCH 011/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ec99a8048..b4996a8ef 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -20,6 +20,7 @@ import ( func SaveModelByParameters(trainTaskId int64, name string, version string, label string, description string, userId int64) { aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ JobID: trainTaskId, + Type: -1, }) if err != nil { log.Info("query task error." + err.Error()) From e36e756f5ed7fc0c82465057818bcc077d37a747 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 16:21:10 +0800 Subject: [PATCH 012/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b4996a8ef..67885777c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -18,7 +18,7 @@ import ( ) func SaveModelByParameters(trainTaskId int64, name string, version string, label string, description string, userId int64) { - aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ + aiTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ JobID: trainTaskId, Type: -1, }) @@ -27,6 +27,8 @@ func SaveModelByParameters(trainTaskId int64, name string, version string, label //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) return } + log.Info("query return count=" + fmt.Sprint(count)) + uuid := uuid.NewV4() id := uuid.String() modelPath := id From b4b54be9f4adc2727b4f5f4a7341b6fcd51157eb Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 16:25:47 +0800 Subject: [PATCH 013/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/tool.go | 2 +- routers/repo/ai_model_manage.go | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/routers/private/tool.go b/routers/private/tool.go index 68de67aa7..3b8322d94 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -46,7 +46,7 @@ func RepoStatisticManually(ctx *macaron.Context) { } func CreateModel(ctx *macaron.Context) { - trainTaskId := ctx.QueryInt64("TrainTask") + trainTaskId := ctx.Query("TrainTask") name := ctx.Query("Name") version := ctx.Query("Version") label := ctx.Query("Label") diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 67885777c..c647483a5 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -17,17 +17,13 @@ import ( uuid "github.com/satori/go.uuid" ) -func SaveModelByParameters(trainTaskId int64, name string, version string, label string, description string, userId int64) { - aiTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ - JobID: trainTaskId, - Type: -1, - }) +func SaveModelByParameters(trainTaskId string, name string, version string, label string, description string, userId int64) { + aiTask, err := models.GetCloudbrainByJobID(trainTaskId) if err != nil { log.Info("query task error." + err.Error()) //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) return } - log.Info("query return count=" + fmt.Sprint(count)) uuid := uuid.NewV4() id := uuid.String() @@ -36,12 +32,6 @@ func SaveModelByParameters(trainTaskId int64, name string, version string, label var modelSize int64 cloudType := models.TypeCloudBrainTwo - if len(aiTasks) != 1 { - log.Info("query task error. len=" + fmt.Sprint(len(aiTasks))) - //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) - return - } - aiTask := aiTasks[0] log.Info("find task name:" + aiTask.JobName) aimodels := models.QueryModelByName(name, userId) if len(aimodels) > 0 { @@ -51,7 +41,7 @@ func SaveModelByParameters(trainTaskId int64, name string, version string, label } } } - cloudType = aiTask.Cloudbrain.Type + cloudType = aiTask.Type //download model zip //train type if cloudType == models.TypeCloudBrainTrainJob { modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") From de9d3d12c93ca4a1b0521c4fc3c7f1a08bb47b68 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:05:46 +0800 Subject: [PATCH 014/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 68 +++++++++++++++++++++++++++++++++++++++++ routers/private/internal.go | 2 ++ routers/private/tool.go | 15 +++++++++ routers/repo/ai_model_manage.go | 46 ++++++++++++++++++++++++++-- 4 files changed, 128 insertions(+), 3 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index b6169b806..7ed6b1051 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -4,7 +4,9 @@ import ( "fmt" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) type AiModelManage struct { @@ -32,6 +34,16 @@ type AiModelManage struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } +type AiModelQueryOptions struct { + ListOptions + RepoID int64 // include all repos if empty + UserID int64 + ModelID string + SortType string + // JobStatus CloudbrainStatus + Type int +} + func SaveModelToDb(model *AiModelManage) error { sess := x.NewSession() defer sess.Close() @@ -68,3 +80,59 @@ func QueryModelByName(name string, uid int64) []*AiModelManage { sess.Find(&aiModelManageList) return aiModelManageList } + +func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.repo_id": opts.RepoID}, + ) + } + + if opts.UserID > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.user_id": opts.UserID}, + ) + } + + if len(opts.ModelID) > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.id": opts.ModelID}, + ) + } + + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.type": opts.Type}, + ) + } + + count, err := sess.Where(cond).Count(new(AiModelManage)) + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("ai_model_manage.created_unix DESC") + aiModelManages := make([]*AiModelManage, 0, setting.UI.IssuePagingNum) + if err := sess.Table(&AiModelManage{}).Where(cond). + Join("left", "`user`", "ai_model_manage.user_id = `user`.id"). + Find(&aiModelManages); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + sess.Close() + + return aiModelManages, count, nil +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 5b1d63bff..29debdfdc 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -45,5 +45,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat", RepoStatisticManually) m.Post("/tool/create_model", CreateModel) + m.Post("/tool/delete_model", DeleteModel) + m.Post("/tool/show_model", ShowModel) }, CheckInternalToken) } diff --git a/routers/private/tool.go b/routers/private/tool.go index 3b8322d94..f3487836b 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -56,3 +56,18 @@ func CreateModel(ctx *macaron.Context) { repo.SaveModelByParameters(trainTaskId, name, version, label, description, userId) } + +func DeleteModel(ctx *macaron.Context) { + id := ctx.Query("id") + repo.DeleteModelByID(id) +} + +func ShowModel(ctx *macaron.Context) { + repoId := ctx.QueryInt64("repoId") + modelResult, _, err := repo.QueryModelByParameters(repoId, 5) + if err == nil { + ctx.JSON(200, modelResult) + } else { + ctx.JSON(500, "query error.") + } +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index c647483a5..6ad37a814 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -43,7 +43,7 @@ func SaveModelByParameters(trainTaskId string, name string, version string, labe } cloudType = aiTask.Type //download model zip //train type - if cloudType == models.TypeCloudBrainTrainJob { + if cloudType == models.TypeCloudBrainTwo { modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") if err == nil { @@ -174,7 +174,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir func DeleteModel(ctx *context.Context) { log.Info("delete model start.") id := ctx.Query("ID") - err := models.DeleteModelById(id) + err := DeleteModelByID(id) if err != nil { ctx.JSON(500, err.Error()) } else { @@ -184,14 +184,54 @@ func DeleteModel(ctx *context.Context) { } } +func DeleteModelByID(id string) error { + log.Info("delete model start. id=" + id) + return models.DeleteModelById(id) +} + func DownloadModel(ctx *context.Context) { log.Info("download model start.") } +func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) { + + return models.QueryModel(&models.AiModelQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repoId, + }) +} + func ShowModelInfo(ctx *context.Context) { - log.Info("ShowModelInfo.") + log.Info("ShowModelInfo start.") + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + repoId := ctx.QueryInt64("repoId") + + modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repoId, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = modelResult + ctx.HTML(200, "") } func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) { From 49fcd64cc09ca2169b06ffdc637d47c0b6a6a7f9 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:09:59 +0800 Subject: [PATCH 015/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/internal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/private/internal.go b/routers/private/internal.go index 29debdfdc..8078bedb0 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -45,7 +45,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat", RepoStatisticManually) m.Post("/tool/create_model", CreateModel) - m.Post("/tool/delete_model", DeleteModel) - m.Post("/tool/show_model", ShowModel) + m.Delete("/tool/delete_model", DeleteModel) + m.Get("/tool/show_model", ShowModel) }, CheckInternalToken) } From 20c679957c5b0df28a47ab19e6cbedd0cf6c4ba9 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:16:02 +0800 Subject: [PATCH 016/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 6ad37a814..28286587f 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -202,6 +202,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in PageSize: setting.UI.IssuePagingNum, }, RepoID: repoId, + Type: -1, }) } @@ -213,13 +214,14 @@ func ShowModelInfo(ctx *context.Context) { page = 1 } repoId := ctx.QueryInt64("repoId") - + Type := -1 modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, }, RepoID: repoId, + Type: Type, }) if err != nil { ctx.ServerError("Cloudbrain", err) From e3e228f8a67c5da3b1f9bb737904c01dde0eb725 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:19:07 +0800 Subject: [PATCH 017/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 3 +-- routers/private/tool.go | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 7ed6b1051..65edcdc46 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -127,8 +127,7 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { sess.OrderBy("ai_model_manage.created_unix DESC") aiModelManages := make([]*AiModelManage, 0, setting.UI.IssuePagingNum) - if err := sess.Table(&AiModelManage{}).Where(cond). - Join("left", "`user`", "ai_model_manage.user_id = `user`.id"). + if err := sess.Table("ai_model_manage").Where(cond). Find(&aiModelManages); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/routers/private/tool.go b/routers/private/tool.go index f3487836b..0e3ea72aa 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -5,6 +5,7 @@ package private import ( + "fmt" "net/http" "code.gitea.io/gitea/models" @@ -64,8 +65,9 @@ func DeleteModel(ctx *macaron.Context) { func ShowModel(ctx *macaron.Context) { repoId := ctx.QueryInt64("repoId") - modelResult, _, err := repo.QueryModelByParameters(repoId, 5) + modelResult, count, err := repo.QueryModelByParameters(repoId, 5) if err == nil { + log.Info("count=" + fmt.Sprint(count)) ctx.JSON(200, modelResult) } else { ctx.JSON(500, "query error.") From 6539e52f5b345a468fbad27405180d8c9b98c967 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:22:04 +0800 Subject: [PATCH 018/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/tool.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routers/private/tool.go b/routers/private/tool.go index 0e3ea72aa..c7d08430b 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -5,6 +5,7 @@ package private import ( + "encoding/json" "fmt" "net/http" @@ -68,7 +69,9 @@ func ShowModel(ctx *macaron.Context) { modelResult, count, err := repo.QueryModelByParameters(repoId, 5) if err == nil { log.Info("count=" + fmt.Sprint(count)) - ctx.JSON(200, modelResult) + modelResultjson, _ := json.Marshal(modelResult) + + ctx.JSON(200, (string(modelResultjson))) } else { ctx.JSON(500, "query error.") } From 944b6f879b79b4a65b71e0e050b4736111b3c76c Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:25:53 +0800 Subject: [PATCH 019/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/tool.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/private/tool.go b/routers/private/tool.go index c7d08430b..d751c38a7 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -5,7 +5,6 @@ package private import ( - "encoding/json" "fmt" "net/http" @@ -66,12 +65,13 @@ func DeleteModel(ctx *macaron.Context) { func ShowModel(ctx *macaron.Context) { repoId := ctx.QueryInt64("repoId") - modelResult, count, err := repo.QueryModelByParameters(repoId, 5) + page := ctx.QueryInt("page") + modelResult, count, err := repo.QueryModelByParameters(repoId, page) if err == nil { log.Info("count=" + fmt.Sprint(count)) - modelResultjson, _ := json.Marshal(modelResult) + //modelResultjson, _ := json.Marshal(modelResult) - ctx.JSON(200, (string(modelResultjson))) + ctx.JSON(200, modelResult) } else { ctx.JSON(500, "query error.") } From 31589e17a02e16afb1243412bdbff2f76d9800a1 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:41:25 +0800 Subject: [PATCH 020/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 28286587f..47b3501a0 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -7,7 +7,9 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" + "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -151,17 +153,18 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir models.AttachmentRelativePath(modelUUID) + "/" - models, err := storage.GetObsListObject(jobName, parentDir) + modelDbResult, err := storage.GetObsListObject(jobName, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) return "", 0, err } - if len(models) == 0 { + if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - - for _, modelFile := range models { - log.Info("copy file, bucket=%s, src keyname=%s,dest keyname=%s", setting.Bucket, modelFile.ParenDir+modelFile.FileName, dataActualPath+modelFile.FileName) + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + for _, modelFile := range modelDbResult { + destKeyName := "/aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + log.Info("copy file, bucket=%s, src keyname=%s, dest keyname=%s,", setting.Bucket, prefix+modelFile.FileName, destKeyName) // err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) // if err != nil { // log.Info("copy failed.") From 3938cef7375a99ac800f1b4f37f9ce5d16a239b4 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 17:49:51 +0800 Subject: [PATCH 021/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 2 +- routers/repo/ai_model_manage.go | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 65edcdc46..1f8953137 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -75,7 +75,7 @@ func QueryModelByName(name string, uid int64) []*AiModelManage { sess := x.NewSession() defer sess.Close() sess.Select("*").Table("ai_model_manage"). - Where("name=" + name + " and user_id=" + fmt.Sprint(uid)) + Where("name='" + name + "' and user_id=" + fmt.Sprint(uid)) aiModelManageList := make([]*AiModelManage, 0) sess.Find(&aiModelManageList) return aiModelManageList diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 47b3501a0..1cbbcb327 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -163,12 +163,13 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir } prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") for _, modelFile := range modelDbResult { - destKeyName := "/aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" - log.Info("copy file, bucket=%s, src keyname=%s, dest keyname=%s,", setting.Bucket, prefix+modelFile.FileName, destKeyName) - // err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) - // if err != nil { - // log.Info("copy failed.") - // } + destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) + log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) + err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) + if err != nil { + log.Info("copy failed.") + } } return dataActualPath, 0, nil From ace944cacedb7749b7c4866e1ae84d88a5c21c7f Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 09:59:46 +0800 Subject: [PATCH 022/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 1cbbcb327..5170c889d 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -166,7 +166,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) - err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) + err := storage.ObsCopyFile(setting.Bucket, prefix+modelFile.FileName, setting.Bucket, destKeyNamePrefix+modelFile.FileName) if err != nil { log.Info("copy failed.") } From 9a212216670099cc22deb61912d2f3174673a72c Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 10:14:10 +0800 Subject: [PATCH 023/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 5170c889d..ad06dc031 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -161,7 +161,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" for _, modelFile := range modelDbResult { destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) From 59bfbd2ac1c45f77b8b97dccde5dc0fb0b3916e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 10:18:44 +0800 Subject: [PATCH 024/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ad06dc031..b85e3ba64 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -161,9 +161,10 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" + prefix := "/" + strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" for _, modelFile := range modelDbResult { - destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + destKeyNamePrefix := "/aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) err := storage.ObsCopyFile(setting.Bucket, prefix+modelFile.FileName, setting.Bucket, destKeyNamePrefix+modelFile.FileName) From cca33572cf4ba12f6d9bd39fcf7e1b3af74a2fff Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 10:21:10 +0800 Subject: [PATCH 025/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 8 ++++---- routers/repo/ai_model_manage.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 87e972572..2f3e379f8 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -176,10 +176,10 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { input := &obs.CopyObjectInput{} - input.Bucket = srcBucket - input.Key = srcKeyName - input.CopySourceBucket = destBucket - input.CopySourceKey = destKeyName + input.Bucket = destBucket + input.Key = destKeyName + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKeyName _, err := ObsCli.CopyObject(input) if err == nil { log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b85e3ba64..830ed9f39 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -161,9 +161,9 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - prefix := "/" + strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" for _, modelFile := range modelDbResult { - destKeyNamePrefix := "/aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) From cbf86080140730e1ee01c8989b4187d658453109 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 11:13:02 +0800 Subject: [PATCH 026/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 10 +++ modules/storage/obs.go | 37 +++++++++ routers/repo/ai_model_manage.go | 163 ++++++---------------------------------- routers/repo/dir.go | 37 +-------- 4 files changed, 73 insertions(+), 174 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 1f8953137..d17c517b8 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -56,6 +56,16 @@ func SaveModelToDb(model *AiModelManage) error { return nil } +func QueryModelById(id string) AiModelManage { + sess := x.NewSession() + defer sess.Close() + sess.Select("*").Table("ai_model_manage"). + Where("id='" + id + "'") + var aiModelManage AiModelManage + sess.Find(&aiModelManage) + return aiModelManage +} + func DeleteModelById(id string) error { sess := x.NewSession() defer sess.Close() diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 2f3e379f8..be36e7e69 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -138,6 +138,43 @@ func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName s } +//delete all file under the dir path +func ObsRemoveObject(bucket string, path string) error { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = path + index := 1 + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + log.Info("delete obs file:" + val.Key) + delObj := &obs.DeleteObjectInput{} + delObj.Bucket = setting.Bucket + delObj.Key = val.Key + ObsCli.DeleteObject(delObj) + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return err + } + } + return nil +} + func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 830ed9f39..61a48b815 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -1,14 +1,9 @@ package repo import ( - "archive/zip" "errors" "fmt" - "io" - "io/ioutil" - "os" "path" - "path/filepath" "strings" "code.gitea.io/gitea/models" @@ -19,12 +14,11 @@ import ( uuid "github.com/satori/go.uuid" ) -func SaveModelByParameters(trainTaskId string, name string, version string, label string, description string, userId int64) { - aiTask, err := models.GetCloudbrainByJobID(trainTaskId) +func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { + aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { log.Info("query task error." + err.Error()) - //ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) - return + return err } uuid := uuid.NewV4() @@ -47,12 +41,9 @@ func SaveModelByParameters(trainTaskId string, name string, version string, labe //download model zip //train type if cloudType == models.TypeCloudBrainTwo { modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") - if err == nil { - - } else { + if err != nil { log.Info("download model from CloudBrainTwo faild." + err.Error()) - //ctx.Error(500, fmt.Sprintf("%v", err)) - return + return err } } @@ -74,75 +65,25 @@ func SaveModelByParameters(trainTaskId string, name string, version string, labe models.SaveModelToDb(model) log.Info("save model end.") + + return nil } func SaveModel(ctx *context.Context) { log.Info("save model start.") - trainTaskId := ctx.QueryInt64("TrainTask") + JobId := ctx.Query("JobId") name := ctx.Query("Name") version := ctx.Query("Version") label := ctx.Query("Label") description := ctx.Query("Description") - aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ - JobID: trainTaskId, - }) - if err != nil { - log.Info("query task error." + err.Error()) - ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) - return - } - uuid := uuid.NewV4() - id := uuid.String() - modelPath := id - parent := id - var modelSize int64 - cloudType := models.TypeCloudBrainTwo + err := SaveModelByParameters(JobId, name, version, label, description, ctx.User.ID) - if len(aiTasks) != 1 { - log.Info("query task error. len=" + fmt.Sprint(len(aiTasks))) - ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err)) + if err != nil { + log.Info("save model error." + err.Error()) + ctx.Error(500, fmt.Sprintf("save model error. %v", err)) return } - aiTask := aiTasks[0] - log.Info("find task name:" + aiTask.JobName) - aimodels := models.QueryModelByName(name, ctx.User.ID) - if len(aimodels) > 0 { - for _, model := range aimodels { - if model.ID == model.Parent { - parent = model.ID - } - } - } - cloudType = aiTask.Cloudbrain.Type - //download model zip //train type - if cloudType == models.TypeCloudBrainTrainJob { - modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") - if err == nil { - - } else { - log.Info("download model from CloudBrainTwo faild." + err.Error()) - ctx.Error(500, fmt.Sprintf("%v", err)) - return - } - } - - model := &models.AiModelManage{ - ID: id, - Version: version, - Label: label, - Name: name, - Description: description, - Parent: parent, - Type: cloudType, - Path: modelPath, - Size: modelSize, - AttachmentId: aiTask.Uuid, - RepoId: aiTask.RepoID, - UserId: ctx.User.ID, - } - - models.SaveModelToDb(model) log.Info("save model end.") } @@ -161,6 +102,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } + var size int64 prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" for _, modelFile := range modelDbResult { destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" @@ -171,9 +113,10 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if err != nil { log.Info("copy failed.") } + size += modelFile.Size } - return dataActualPath, 0, nil + return dataActualPath, size, nil } func DeleteModel(ctx *context.Context) { @@ -191,6 +134,14 @@ func DeleteModel(ctx *context.Context) { func DeleteModelByID(id string) error { log.Info("delete model start. id=" + id) + + model := models.QueryModelById(id) + + err := storage.ObsRemoveObject(setting.Bucket, model.Path) + if err != nil { + log.Info("Failed to delete model. id=" + id) + return err + } return models.DeleteModelById(id) } @@ -240,71 +191,3 @@ func ShowModelInfo(ctx *context.Context) { ctx.Data["Tasks"] = modelResult ctx.HTML(200, "") } - -func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) { - - modelActualPath := setting.Attachment.Minio.RealPath + - setting.Attachment.Minio.Bucket + "/" + - "aimodels/" + - models.AttachmentRelativePath(modelUUID) + - "/" - os.MkdirAll(modelActualPath, 0755) - zipFile := modelActualPath + "model.zip" - - modelDir := setting.JobPath + jobName + "/model/" - - dir, _ := ioutil.ReadDir(modelDir) - if len(dir) == 0 { - return "", 0, errors.New("cannot create model, as model is empty.") - } - - err := zipDir(modelDir, zipFile) - if err != nil { - return "", 0, err - } - - fi, err := os.Stat(zipFile) - if err == nil { - return modelActualPath, fi.Size(), nil - } else { - return "", 0, err - } -} - -func zipDir(dir, zipFile string) error { - fz, err := os.Create(zipFile) - if err != nil { - log.Info("Create zip file failed: %s\n", err.Error()) - return err - } - defer fz.Close() - - w := zip.NewWriter(fz) - defer w.Close() - - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() { - fDest, err := w.Create(path[len(dir)+1:]) - if err != nil { - log.Info("Create failed: %s\n", err.Error()) - return err - } - fSrc, err := os.Open(path) - if err != nil { - log.Info("Open failed: %s\n", err.Error()) - return err - } - defer fSrc.Close() - _, err = io.Copy(fDest, fSrc) - if err != nil { - log.Info("Copy failed: %s\n", err.Error()) - return err - } - } - return nil - }) - if err != nil { - return err - } - return nil -} diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 406f3dc73..81549e76a 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -70,40 +69,10 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) { } } if attachment.Type == models.TypeCloudBrainTwo { - - input := &obs.ListObjectsInput{} - input.Bucket = setting.Bucket - // 设置每页100个对象 - input.MaxKeys = 100 - input.Prefix = setting.BasePath + attachment.RelativePath() + attachment.UUID - index := 1 - log.Info("prefix=" + input.Prefix) - for { - output, err := storage.ObsCli.ListObjects(input) - if err == nil { - log.Info("Page:%d\n", index) - index++ - for _, val := range output.Contents { - log.Info("delete obs file:" + val.Key) - delObj := &obs.DeleteObjectInput{} - delObj.Bucket = setting.Bucket - delObj.Key = val.Key - storage.ObsCli.DeleteObject(delObj) - } - if output.IsTruncated { - input.Marker = output.NextMarker - } else { - break - } - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Info("Code:%s\n", obsError.Code) - log.Info("Message:%s\n", obsError.Message) - } - break - } + err := storage.ObsRemoveObject(setting.Bucket, setting.BasePath+attachment.RelativePath()+attachment.UUID) + if err != nil { + log.Info("delete file error.") } - } } From 23cc187ee446ec2216476e77c25097a3b714a1a8 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 11:25:08 +0800 Subject: [PATCH 027/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 61a48b815..e1a5022fb 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -136,12 +136,12 @@ func DeleteModelByID(id string) error { log.Info("delete model start. id=" + id) model := models.QueryModelById(id) - - err := storage.ObsRemoveObject(setting.Bucket, model.Path) - if err != nil { - log.Info("Failed to delete model. id=" + id) - return err - } + log.Info("bucket=" + setting.Bucket + " path=" + model.Path) + //err := storage.ObsRemoveObject(setting.Bucket, model.Path) + //if err != nil { + // log.Info("Failed to delete model. id=" + id) + // return err + //} return models.DeleteModelById(id) } From 3ee355d2d320c25c308d882aeed7ec65ee5449d1 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 11:48:41 +0800 Subject: [PATCH 028/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 13 +++++++++---- routers/repo/ai_model_manage.go | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index d17c517b8..bdb9efb40 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -56,14 +56,19 @@ func SaveModelToDb(model *AiModelManage) error { return nil } -func QueryModelById(id string) AiModelManage { +func QueryModelById(id string) (*AiModelManage, error) { sess := x.NewSession() defer sess.Close() sess.Select("*").Table("ai_model_manage"). Where("id='" + id + "'") - var aiModelManage AiModelManage - sess.Find(&aiModelManage) - return aiModelManage + aiModelManageList := make([]*AiModelManage, 0) + err := sess.Find(&aiModelManageList) + if err != nil { + if len(aiModelManageList) == 1 { + return aiModelManageList[0], nil + } + } + return nil, err } func DeleteModelById(id string) error { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index e1a5022fb..859a2f7b8 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -135,14 +135,17 @@ func DeleteModel(ctx *context.Context) { func DeleteModelByID(id string) error { log.Info("delete model start. id=" + id) - model := models.QueryModelById(id) - log.Info("bucket=" + setting.Bucket + " path=" + model.Path) - //err := storage.ObsRemoveObject(setting.Bucket, model.Path) - //if err != nil { - // log.Info("Failed to delete model. id=" + id) - // return err - //} - return models.DeleteModelById(id) + model, err := models.QueryModelById(id) + if err == nil { + log.Info("bucket=" + setting.Bucket + " path=" + model.Path) + //err := storage.ObsRemoveObject(setting.Bucket, model.Path) + //if err != nil { + // log.Info("Failed to delete model. id=" + id) + // return err + //} + return models.DeleteModelById(id) + } + return err } func DownloadModel(ctx *context.Context) { From 6150d5e3dd8a9f24fc09f7c9dabab0e3cb09cf1f Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 14:17:47 +0800 Subject: [PATCH 029/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index be36e7e69..e78c07708 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -140,6 +140,10 @@ func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName s //delete all file under the dir path func ObsRemoveObject(bucket string, path string) error { + log.Info("Bucket=" + bucket + " path=" + path) + if len(path) == 0 { + return error.Error("path cannot be null.") + } input := &obs.ListObjectsInput{} input.Bucket = bucket // 设置每页100个对象 @@ -157,7 +161,7 @@ func ObsRemoveObject(bucket string, path string) error { delObj := &obs.DeleteObjectInput{} delObj.Bucket = setting.Bucket delObj.Key = val.Key - ObsCli.DeleteObject(delObj) + //ObsCli.DeleteObject(delObj) } if output.IsTruncated { input.Marker = output.NextMarker From 696836fdfde482dde77cb8c302a4588f536898e8 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 14:22:53 +0800 Subject: [PATCH 030/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 3 ++- routers/repo/ai_model_manage.go | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e78c07708..6636f0976 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,6 +5,7 @@ package storage import ( + "errors" "io" "path" "strconv" @@ -142,7 +143,7 @@ func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName s func ObsRemoveObject(bucket string, path string) error { log.Info("Bucket=" + bucket + " path=" + path) if len(path) == 0 { - return error.Error("path cannot be null.") + return errors.New("path canot be null.") } input := &obs.ListObjectsInput{} input.Bucket = bucket diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 859a2f7b8..f3e1a4439 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -14,6 +14,10 @@ import ( uuid "github.com/satori/go.uuid" ) +const ( + Model_prefix = "aimodels/" +) + func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { @@ -89,8 +93,7 @@ func SaveModel(ctx *context.Context) { } func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - dataActualPath := setting.Bucket + "/" + - "aimodels/" + + dataActualPath := setting.Bucket + "/" + Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" @@ -105,7 +108,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir var size int64 prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" for _, modelFile := range modelDbResult { - destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/" + destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) @@ -138,11 +141,14 @@ func DeleteModelByID(id string) error { model, err := models.QueryModelById(id) if err == nil { log.Info("bucket=" + setting.Bucket + " path=" + model.Path) - //err := storage.ObsRemoveObject(setting.Bucket, model.Path) - //if err != nil { - // log.Info("Failed to delete model. id=" + id) - // return err - //} + if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { + err := storage.ObsRemoveObject(setting.Bucket, model.Path[len(setting.Bucket+"/"+Model_prefix):]) + if err != nil { + log.Info("Failed to delete model. id=" + id) + return err + } + } + return models.DeleteModelById(id) } return err From 51207b667f4cb95428498cf6120e78829c024284 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 16:01:20 +0800 Subject: [PATCH 031/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 2 +- routers/repo/ai_model_manage.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index bdb9efb40..404a76282 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -63,7 +63,7 @@ func QueryModelById(id string) (*AiModelManage, error) { Where("id='" + id + "'") aiModelManageList := make([]*AiModelManage, 0) err := sess.Find(&aiModelManageList) - if err != nil { + if err == nil { if len(aiModelManageList) == 1 { return aiModelManageList[0], nil } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index f3e1a4439..e6c08c0ac 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -156,7 +156,6 @@ func DeleteModelByID(id string) error { func DownloadModel(ctx *context.Context) { log.Info("download model start.") - } func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) { From 2c067067aee0e998f8a3849b3091d522d87e32b6 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 5 Nov 2021 16:05:04 +0800 Subject: [PATCH 032/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 2 +- routers/repo/ai_model_manage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 6636f0976..df12c73e3 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -162,7 +162,7 @@ func ObsRemoveObject(bucket string, path string) error { delObj := &obs.DeleteObjectInput{} delObj.Bucket = setting.Bucket delObj.Key = val.Key - //ObsCli.DeleteObject(delObj) + ObsCli.DeleteObject(delObj) } if output.IsTruncated { input.Marker = output.NextMarker diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index e6c08c0ac..307129911 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -142,7 +142,7 @@ func DeleteModelByID(id string) error { if err == nil { log.Info("bucket=" + setting.Bucket + " path=" + model.Path) if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { - err := storage.ObsRemoveObject(setting.Bucket, model.Path[len(setting.Bucket+"/"+Model_prefix):]) + err := storage.ObsRemoveObject(setting.Bucket, model.Path[len(setting.Bucket)+1:]) if err != nil { log.Info("Failed to delete model. id=" + id) return err From 1b9c667271787d09ef9f64c1a7077c49ea7f6733 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 09:10:03 +0800 Subject: [PATCH 033/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 16 ++++++++++++++++ routers/private/internal.go | 1 + routers/private/tool.go | 11 +++++++++++ routers/repo/ai_model_manage.go | 25 +++++++++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 404a76282..310e51019 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -86,6 +86,22 @@ func DeleteModelById(id string) error { } +func ModifyModelDescription(id string, description string) error { + sess := x.NewSession() + defer sess.Close() + + re, err := sess.Update(&AiModelManage{ + ID: id, + Description: description, + }) + if err != nil { + return err + } + log.Info("success to update description from db.re=" + fmt.Sprint((re))) + return nil + +} + func QueryModelByName(name string, uid int64) []*AiModelManage { sess := x.NewSession() defer sess.Close() diff --git a/routers/private/internal.go b/routers/private/internal.go index 8078bedb0..a3ea1c6cd 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -47,5 +47,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/tool/create_model", CreateModel) m.Delete("/tool/delete_model", DeleteModel) m.Get("/tool/show_model", ShowModel) + m.Put("/tool/modify_model", ModifyModel) }, CheckInternalToken) } diff --git a/routers/private/tool.go b/routers/private/tool.go index d751c38a7..bccbc50f3 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -76,3 +76,14 @@ func ShowModel(ctx *macaron.Context) { ctx.JSON(500, "query error.") } } + +func ModifyModel(ctx *macaron.Context) { + id := ctx.Query("id") + description := ctx.Query("Description") + err := repo.ModifyModel(id, description) + if err == nil { + ctx.JSON(200, "Success.") + } else { + ctx.JSON(500, "Failed.") + } +} diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 307129911..321b3a023 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -199,3 +199,28 @@ func ShowModelInfo(ctx *context.Context) { ctx.Data["Tasks"] = modelResult ctx.HTML(200, "") } + +func ModifyModel(id string, description string) error { + err := models.ModifyModelDescription(id, description) + if err == nil { + log.Info("modify success.") + } else { + log.Info("Failed to modify.id=" + id + " desc=" + description) + } + return err +} + +func ModifyModelInfo(ctx *context.Context) { + log.Info("delete model start.") + id := ctx.Query("ID") + description := ctx.Query("Description") + + err := ModifyModel(id, description) + + if err == nil { + ctx.HTML(200, "success") + } else { + ctx.HTML(500, "Failed.") + } + +} From fba7096e831f4404e965d60777ab82107af9cf77 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 09:46:40 +0800 Subject: [PATCH 034/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 8 ++++---- routers/repo/ai_model_manage.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 310e51019..cab4201c7 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -7,6 +7,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) type AiModelManage struct { @@ -87,11 +88,10 @@ func DeleteModelById(id string) error { } func ModifyModelDescription(id string, description string) error { - sess := x.NewSession() + var sess *xorm.Session + sess = x.ID(id) defer sess.Close() - - re, err := sess.Update(&AiModelManage{ - ID: id, + re, err := sess.Cols("description").Update(&AiModelManage{ Description: description, }) if err != nil { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 321b3a023..47d8e67a4 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -205,7 +205,7 @@ func ModifyModel(id string, description string) error { if err == nil { log.Info("modify success.") } else { - log.Info("Failed to modify.id=" + id + " desc=" + description) + log.Info("Failed to modify.id=" + id + " desc=" + description + " error:" + err.Error()) } return err } From c5e5f544157893afd0793504e7215f49d196633c Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 15:23:30 +0800 Subject: [PATCH 035/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 92 ++++++++- routers/repo/ai_model_manage.go | 51 ++++- routers/repo/modelarts.go | 8 +- routers/routes/routes.go | 3 + templates/repo/modelarts/index.tmpl | 1 + templates/repo/modelmanage/index.tmpl | 375 ++++++++++++++++++++++++++++++++++ 6 files changed, 512 insertions(+), 18 deletions(-) create mode 100644 templates/repo/modelmanage/index.tmpl diff --git a/modules/storage/obs.go b/modules/storage/obs.go index df12c73e3..76e979967 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -180,11 +180,10 @@ func ObsRemoveObject(bucket string, path string) error { return nil } -func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { +func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} - input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") - // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + input.Bucket = bucket + input.Key = key output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", @@ -198,6 +197,11 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { } } +func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { + + return ObsDownloadAFile(setting.Bucket, strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")) +} + func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket @@ -235,6 +239,48 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } +func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefix + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + if err == nil { + for _, val := range output.Contents { + str1 := strings.Split(val.Key, "/") + var isDir bool + var fileName, nextParentDir string + if strings.HasSuffix(val.Key, "/") { + fileName = str1[len(str1)-2] + isDir = true + nextParentDir = fileName + if (fileName + "/") == setting.OutPutPath { + continue + } + } else { + fileName = str1[len(str1)-1] + isDir = false + } + + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: nextParentDir, + } + fileInfos = append(fileInfos, fileInfo) + } + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } + +} + func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket @@ -299,16 +345,19 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file return output.SignedUrl, nil } -func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { +func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { input := &obs.CreateSignedUrlInput{} - input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") - + input.Bucket = bucket + input.Key = key input.Expires = 60 * 60 input.Method = obs.HttpMethodGet - + comma := strings.LastIndex(key, "/") + filename := key + if comma != -1 { + filename = key[comma+1:] + } reqParams := make(map[string]string) - reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" + reqParams["response-content-disposition"] = "attachment; filename=\"" + filename + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { @@ -317,6 +366,29 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) } return output.SignedUrl, nil + +} + +func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { + // input := &obs.CreateSignedUrlInput{} + // input.Bucket = setting.Bucket + // input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") + + return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")) + + // input.Expires = 60 * 60 + // input.Method = obs.HttpMethodGet + + // reqParams := make(map[string]string) + // reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" + // input.QueryParams = reqParams + // output, err := ObsCli.CreateSignedUrl(input) + // if err != nil { + // log.Error("CreateSignedUrl failed:", err.Error()) + // return "", err + // } + + // return output.SignedUrl, nil } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 47d8e67a4..b1ebe9478 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -3,6 +3,7 @@ package repo import ( "errors" "fmt" + "net/http" "path" "strings" @@ -15,7 +16,8 @@ import ( ) const ( - Model_prefix = "aimodels/" + Model_prefix = "aimodels/" + tplModelManageIndex = "repo/modelmanage/index" ) func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { @@ -170,14 +172,55 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in }) } -func ShowModelInfo(ctx *context.Context) { +func DownloadMultiModelFile(ctx *context.Context) { + log.Info("DownloadMultiModelFile start.") + id := ctx.Query("ID") + log.Info("id=" + id) +} + +func DownloadSingleModelFile(ctx *context.Context) { + log.Info("DownloadSingleModelFile start.") + path := ctx.Query("path") + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path[len(setting.Bucket)+1:]) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +} + +func ShowSingleModel(ctx *context.Context) { + log.Info("Show single ModelInfo start.") + id := ctx.Query("ID") + task, err := models.QueryModelById(id) + if err != nil { + log.Error("no such model!", ctx.Data["msgID"]) + ctx.ServerError("no such model:", err) + return + } + + models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:]) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.Data["Dirs"] = models + ctx.Data["task"] = task + ctx.Data["ID"] = id + +} + +func ShowModelPageInfo(ctx *context.Context) { log.Info("ShowModelInfo start.") page := ctx.QueryInt("page") if page <= 0 { page = 1 } - repoId := ctx.QueryInt64("repoId") + repoId := ctx.Repo.Repository.ID Type := -1 modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{ ListOptions: models.ListOptions{ @@ -197,7 +240,7 @@ func ShowModelInfo(ctx *context.Context) { ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = modelResult - ctx.HTML(200, "") + ctx.HTML(200, tplModelManageIndex) } func ModifyModel(id string, description string) error { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index e5bd47aac..592459f13 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -286,8 +286,8 @@ func NotebookIndex(ctx *context.Context) { Page: page, PageSize: setting.UI.IssuePagingNum, }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, JobType: string(models.JobTypeDebug), }) if err != nil { @@ -512,8 +512,8 @@ func TrainJobIndex(ctx *context.Context) { Page: page, PageSize: setting.UI.IssuePagingNum, }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, JobType: string(models.JobTypeTrain), }) if err != nil { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 7819e3930..366ae75d8 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -960,6 +960,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) + m.Group("/modelmanage", func() { + m.Get("", reqRepoCloudBrainReader, repo.ShowModelPageInfo) + }, context.RepoRef()) m.Group("/modelarts", func() { // m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex) diff --git a/templates/repo/modelarts/index.tmpl b/templates/repo/modelarts/index.tmpl index 5ceccdd9a..d9bdb2e10 100755 --- a/templates/repo/modelarts/index.tmpl +++ b/templates/repo/modelarts/index.tmpl @@ -209,6 +209,7 @@
调试任务 训练任务 + 模型管理
diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl new file mode 100644 index 000000000..a99031e3a --- /dev/null +++ b/templates/repo/modelmanage/index.tmpl @@ -0,0 +1,375 @@ + +{{template "base/head" .}} + + + + +
+
+
+
+
+
+
+
+
+ + +
+ +
+ {{template "repo/header" .}} + +
+ {{template "base/alert" .}} + +
+
+
+ 调试任务 + 训练任务 + 模型管理 +
+
+
+ +
+ {{svg "octicon-server" 16}} +
Ascend NPU
+ +
+ CPU / GPU + Ascend NPU +
+
+ + 新建调试任务 +
+ +
+ + +
+
+
+ + + + + +
+ + +
+
+
+ {{$.i18n.Tr "repo.cloudbrain_task"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_status_createtime"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_status_runtime"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_creator"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_operate"}} +
+ +
+ +
+ + + + {{range .Tasks}} +
+
+ + +
+ + {{svg "octicon-tasklist" 16}} + {{.Name}} + +
+ +
+ + + + {{.Status}} + + + {{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}} +
+ +
+ + + +
+ + +
+ +
+ + + {{$.i18n.Tr "repo.model_download"}} + + + + +
+ +
+ {{$.CsrfTokenHtml}} + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.delete"}} + + {{else}} + + {{$.i18n.Tr "repo.delete"}} + + {{end}} +
+
+ + + + +
+
+ {{end}} {{template "base/paginate" .}} +
+ +
+
+
+ +
+ +
+
+ + + + +
+
+
+ 删除任务 +
+ +
+

你确认删除该任务么？此任务一旦删除不可恢复。

+
+
+
+ 取消操作 +
+
+ 确定操作 +
+
+
+
+ + +{{template "base/footer" .}} + From b18aad2497c9ec37af89c5bf08a4e9afaa8c3313 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 15:43:21 +0800 Subject: [PATCH 036/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelarts/index.tmpl | 1 - templates/repo/modelarts/trainjob/index.tmpl | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelarts/index.tmpl b/templates/repo/modelarts/index.tmpl index d9bdb2e10..5ceccdd9a 100755 --- a/templates/repo/modelarts/index.tmpl +++ b/templates/repo/modelarts/index.tmpl @@ -209,7 +209,6 @@
调试任务 训练任务 - 模型管理
diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 1453da9e5..05a45d250 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -234,6 +234,7 @@
{{$.i18n.Tr "repo.modelarts.notebook"}} {{$.i18n.Tr "repo.modelarts.train_job"}} + 模型管理
From 5e9d2e35cfb4660f6845c369b69c74ac797351ac Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 15:45:31 +0800 Subject: [PATCH 037/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/index.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index a99031e3a..a60c8fda3 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -294,7 +294,7 @@ {{.Status}} - {{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}} + {{TimeSinceUnix .CreatedUnix $.Lang}}
From c56b5ec87bf43245cba94dc92c00b4cba448a74f Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:11:08 +0800 Subject: [PATCH 038/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 8 +++-- templates/repo/modelmanage/download.tmpl | 59 ++++++++++++++++++++++++++++++++ templates/repo/modelmanage/index.tmpl | 2 +- 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 templates/repo/modelmanage/download.tmpl diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b1ebe9478..2139df236 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -16,8 +16,9 @@ import ( ) const ( - Model_prefix = "aimodels/" - tplModelManageIndex = "repo/modelmanage/index" + Model_prefix = "aimodels/" + tplModelManageIndex = "repo/modelmanage/index" + tplModelManageDownload = "repo/modelmanage/download" ) func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { @@ -210,6 +211,7 @@ func ShowSingleModel(ctx *context.Context) { ctx.Data["Dirs"] = models ctx.Data["task"] = task ctx.Data["ID"] = id + ctx.HTML(200, tplModelManageDownload) } @@ -240,7 +242,7 @@ func ShowModelPageInfo(ctx *context.Context) { ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = modelResult - ctx.HTML(200, tplModelManageIndex) + } func ModifyModel(id string, description string) error { diff --git a/templates/repo/modelmanage/download.tmpl b/templates/repo/modelmanage/download.tmpl new file mode 100644 index 000000000..495e2e371 --- /dev/null +++ b/templates/repo/modelmanage/download.tmpl @@ -0,0 +1,59 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+
+

+ {{ range $index, $item := .Path }}{{ $item }}/{{ end }} +

+
+
+
+ +
+
+
+
+ + 下载所有 + + {{if .Dirs}} + + + {{range .Dirs}} + + + + + + {{end}} + +
+ + + + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} + + + + + {{.Size | FileSize}} + + + {{.ModTime}} +
+ + {{end}} + +
+
+
+
+
+
+ + + +{{template "base/footer" .}} diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index a60c8fda3..3be22ca9e 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -308,7 +308,7 @@
- + {{$.i18n.Tr "repo.model_download"}} From 57533f630deb1c765c1b9576739bbf0843d78298 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:17:10 +0800 Subject: [PATCH 039/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/routes/routes.go | 3 +++ templates/repo/modelmanage/index.tmpl | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 366ae75d8..8c37395a0 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -962,6 +962,9 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelmanage", func() { m.Get("", reqRepoCloudBrainReader, repo.ShowModelPageInfo) + m.Group("/:ID", func() { + m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) + }) }, context.RepoRef()) m.Group("/modelarts", func() { diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 3be22ca9e..4eb4a96a7 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -312,8 +312,6 @@ {{$.i18n.Tr "repo.model_download"}} - -
From d0686b0ddd8310d012fbc98f3170ba8a64b4e4e5 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:21:17 +0800 Subject: [PATCH 040/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 2139df236..71494340f 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -243,6 +243,8 @@ func ShowModelPageInfo(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = modelResult + ctx.HTML(200, tplModelManageIndex) + } func ModifyModel(id string, description string) error { From 949a68ece5532291fce856656f9f7c6aea7f40fc Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:32:25 +0800 Subject: [PATCH 041/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 71494340f..45ac90e63 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -192,11 +192,12 @@ func DownloadSingleModelFile(ctx *context.Context) { } func ShowSingleModel(ctx *context.Context) { + + id := ctx.Params(":ID") log.Info("Show single ModelInfo start.") - id := ctx.Query("ID") task, err := models.QueryModelById(id) if err != nil { - log.Error("no such model!", ctx.Data["msgID"]) + log.Error("no such model!", err.Error()) ctx.ServerError("no such model:", err) return } From 326f4432b823466cb19a563b42798c3a1a41605f Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:43:14 +0800 Subject: [PATCH 042/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 45ac90e63..2ec0151c4 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -192,9 +192,8 @@ func DownloadSingleModelFile(ctx *context.Context) { } func ShowSingleModel(ctx *context.Context) { - id := ctx.Params(":ID") - log.Info("Show single ModelInfo start.") + log.Info("Show single ModelInfo start.id=" + id) task, err := models.QueryModelById(id) if err != nil { log.Error("no such model!", err.Error()) @@ -204,9 +203,11 @@ func ShowSingleModel(ctx *context.Context) { models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:]) if err != nil { - log.Info("get TrainJobListModel failed:", err) + log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) return + } else { + log.Info("get model file,size=%s", len(models)) } ctx.Data["Dirs"] = models From f800487b8435fa4ab066975210c4045702d25dd1 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 16:46:48 +0800 Subject: [PATCH 043/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 2ec0151c4..d95a9db0f 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -200,14 +200,14 @@ func ShowSingleModel(ctx *context.Context) { ctx.ServerError("no such model:", err) return } - + log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:]) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) return } else { - log.Info("get model file,size=%s", len(models)) + log.Info("get model file,size=" + fmt.Sprint(models)) } ctx.Data["Dirs"] = models From 53e8fa0ffe96c8d5c6b67c5e2c4368aef702b72a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 17:05:08 +0800 Subject: [PATCH 044/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 18 +++++++++++------- templates/repo/modelmanage/download.tmpl | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index d95a9db0f..692c34774 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -110,12 +110,17 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir } var size int64 prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" + destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" for _, modelFile := range modelDbResult { - destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" - - log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + prefix + modelFile.FileName) - log.Info("Dest key name=" + destKeyNamePrefix + modelFile.FileName) - err := storage.ObsCopyFile(setting.Bucket, prefix+modelFile.FileName, setting.Bucket, destKeyNamePrefix+modelFile.FileName) + if modelFile.IsDir { + log.Info("copy dir, continue. dir=" + modelFile.FileName) + continue + } + srcKeyName := prefix + modelFile.ParenDir + modelFile.FileName + log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) + destKeyName := destKeyNamePrefix + modelFile.ParenDir + modelFile.FileName + log.Info("Dest key name=" + destKeyName) + err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) if err != nil { log.Info("copy failed.") } @@ -207,14 +212,13 @@ func ShowSingleModel(ctx *context.Context) { ctx.ServerError("GetObsListObject:", err) return } else { - log.Info("get model file,size=" + fmt.Sprint(models)) + log.Info("get model file,size=" + fmt.Sprint(len(models))) } ctx.Data["Dirs"] = models ctx.Data["task"] = task ctx.Data["ID"] = id ctx.HTML(200, tplModelManageDownload) - } func ShowModelPageInfo(ctx *context.Context) { diff --git a/templates/repo/modelmanage/download.tmpl b/templates/repo/modelmanage/download.tmpl index 495e2e371..235529859 100644 --- a/templates/repo/modelmanage/download.tmpl +++ b/templates/repo/modelmanage/download.tmpl @@ -18,7 +18,7 @@
下载所有 - + {{if .Dirs}} From acf0ccfff4416ad27365fa034b05dfec65830ece Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 17:09:57 +0800 Subject: [PATCH 045/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 76e979967..b78439854 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -292,6 +292,7 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { str1 := strings.Split(val.Key, "/") var isDir bool var fileName, nextParentDir string + log.Info("val.Key=" + val.Key) if strings.HasSuffix(val.Key, "/") { fileName = str1[len(str1)-2] isDir = true From eee11cf1ce0d4fe4200ea76a55c88da7f286a2df Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 17:11:36 +0800 Subject: [PATCH 046/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index b78439854..b6083af7b 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -297,6 +297,7 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { fileName = str1[len(str1)-2] isDir = true nextParentDir = fileName + log.Info("nextParentDir1=" + nextParentDir) if fileName == parentDir || (fileName+"/") == setting.OutPutPath { continue } @@ -304,7 +305,7 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { fileName = str1[len(str1)-1] isDir = false } - + log.Info("nextParentDir2=" + nextParentDir) fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, From deb1f242b1feb755082172c728de4f539ae53779 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 17:38:11 +0800 Subject: [PATCH 047/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 47 ++++++++++++++++++++++++++++++++++------- routers/repo/ai_model_manage.go | 4 ++-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index b6083af7b..123ff4094 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -239,6 +239,39 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } +func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefix + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + if err == nil { + for _, val := range output.Contents { + var isDir bool + if strings.HasSuffix(val.Key, "/") { + continue + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: val.Key[len(prefix):], + Size: val.Size, + IsDir: isDir, + ParenDir: "", + } + fileInfos = append(fileInfos, fileInfo) + } + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } + +} + func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket @@ -246,17 +279,18 @@ func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { + var nextParentDir string for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool - var fileName, nextParentDir string + var fileName string if strings.HasSuffix(val.Key, "/") { - fileName = str1[len(str1)-2] - isDir = true - nextParentDir = fileName - if (fileName + "/") == setting.OutPutPath { + if prefix == val.Key { continue } + fileName = str1[len(str1)-2] + isDir = true + nextParentDir = nextParentDir + fileName + "/" } else { fileName = str1[len(str1)-1] isDir = false @@ -292,12 +326,10 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { str1 := strings.Split(val.Key, "/") var isDir bool var fileName, nextParentDir string - log.Info("val.Key=" + val.Key) if strings.HasSuffix(val.Key, "/") { fileName = str1[len(str1)-2] isDir = true nextParentDir = fileName - log.Info("nextParentDir1=" + nextParentDir) if fileName == parentDir || (fileName+"/") == setting.OutPutPath { continue } @@ -305,7 +337,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { fileName = str1[len(str1)-1] isDir = false } - log.Info("nextParentDir2=" + nextParentDir) fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 692c34774..b63763fa3 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -99,8 +99,8 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir dataActualPath := setting.Bucket + "/" + Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" - - modelDbResult, err := storage.GetObsListObject(jobName, parentDir) + objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey) if err != nil { log.Info("get TrainJobListModel failed:", err) return "", 0, err From 0988d71adb2da471c08370de70f6baf0aa39e980 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 8 Nov 2021 17:40:02 +0800 Subject: [PATCH 048/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 123ff4094..e7de0c77d 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -255,7 +255,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro } fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), - FileName: val.Key[len(prefix):], + FileName: val.Key[len(prefix)+1:], Size: val.Size, IsDir: isDir, ParenDir: "", From 32a846f928cbdcaa1532d45a21ee03a403eb75c3 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 11:38:15 +0800 Subject: [PATCH 049/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 27 ++++++++++++-- modules/storage/obs.go | 57 +++++++++++++++++++++++----- routers/repo/ai_model_manage.go | 83 +++++++++++++++++++++++++++-------------- 3 files changed, 127 insertions(+), 40 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index cab4201c7..dfeaba287 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -14,7 +14,7 @@ type AiModelManage struct { ID string `xorm:"pk"` Name string `xorm:"NOT NULL"` Version string `xorm:"NOT NULL"` - Parent string `xorm:"NOT NULL"` + New int `xorm:"NOT NULL"` Type int `xorm:"NOT NULL"` Size int64 `xorm:"NOT NULL"` Description string `xorm:"varchar(2000)"` @@ -41,6 +41,7 @@ type AiModelQueryOptions struct { UserID int64 ModelID string SortType string + New int // JobStatus CloudbrainStatus Type int } @@ -99,14 +100,28 @@ func ModifyModelDescription(id string, description string) error { } log.Info("success to update description from db.re=" + fmt.Sprint((re))) return nil +} + +func ModifyModelNewProperty(id string, new int) error { + var sess *xorm.Session + sess = x.ID(id) + defer sess.Close() + re, err := sess.Cols("new").Update(&AiModelManage{ + New: new, + }) + if err != nil { + return err + } + log.Info("success to update new property from db.re=" + fmt.Sprint((re))) + return nil } -func QueryModelByName(name string, uid int64) []*AiModelManage { +func QueryModelByName(name string, repoId int64) []*AiModelManage { sess := x.NewSession() defer sess.Close() sess.Select("*").Table("ai_model_manage"). - Where("name='" + name + "' and user_id=" + fmt.Sprint(uid)) + Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("version desc") aiModelManageList := make([]*AiModelManage, 0) sess.Find(&aiModelManageList) return aiModelManageList @@ -129,6 +144,12 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { ) } + if opts.New >= 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.new": opts.New}, + ) + } + if len(opts.ModelID) > 0 { cond = cond.And( builder.Eq{"ai_model_manage.id": opts.ModelID}, diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e7de0c77d..a0fa56407 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -220,6 +220,47 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } +func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPath string) (int64, error) { + input := &obs.ListObjectsInput{} + input.Bucket = srcBucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = srcPath + index := 1 + length := len(srcPath) + var fileTotalSize int64 + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + if strings.HasSuffix(val.Key, "/") { + log.Info("copy file, src key=" + val.Key + " is dir, not copy.") + continue + } + destKey := destPath + val.Key[length:] + log.Info("copy file,bucket=" + srcBucket + " src key=" + val.Key + "destbucket=" + destBucket + " dest key=" + destKey) + ObsCopyFile(srcBucket, val.Key, destBucket, destKey) + fileTotalSize += val.Size + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return 0, err + } + } + return fileTotalSize, nil +} + func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { input := &obs.CopyObjectInput{} input.Bucket = destBucket @@ -249,7 +290,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro for _, val := range output.Contents { var isDir bool if strings.HasSuffix(val.Key, "/") { - continue + isDir = true } else { isDir = false } @@ -272,30 +313,28 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro } -func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { +func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefix output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { - var nextParentDir string for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool - var fileName string + var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { - if prefix == val.Key { - continue - } fileName = str1[len(str1)-2] isDir = true - nextParentDir = nextParentDir + fileName + "/" + nextParentDir = fileName + if fileName == parentDir || (fileName+"/") == setting.OutPutPath { + continue + } } else { fileName = str1[len(str1)-1] isDir = false } - fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b63763fa3..3b85a0b20 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -19,6 +19,8 @@ const ( Model_prefix = "aimodels/" tplModelManageIndex = "repo/modelmanage/index" tplModelManageDownload = "repo/modelmanage/download" + MODEL_LATEST = 1 + MODEL_NOT_LATEST = 0 ) func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { @@ -31,16 +33,16 @@ func SaveModelByParameters(jobId string, name string, version string, label stri uuid := uuid.NewV4() id := uuid.String() modelPath := id - parent := id + var lastNewModelId string var modelSize int64 cloudType := models.TypeCloudBrainTwo log.Info("find task name:" + aiTask.JobName) - aimodels := models.QueryModelByName(name, userId) + aimodels := models.QueryModelByName(name, aiTask.RepoID) if len(aimodels) > 0 { for _, model := range aimodels { - if model.ID == model.Parent { - parent = model.ID + if model.New == MODEL_LATEST { + lastNewModelId = model.ID } } } @@ -60,7 +62,7 @@ func SaveModelByParameters(jobId string, name string, version string, label stri Label: label, Name: name, Description: description, - Parent: parent, + New: MODEL_LATEST, Type: cloudType, Path: modelPath, Size: modelSize, @@ -71,6 +73,11 @@ func SaveModelByParameters(jobId string, name string, version string, label stri models.SaveModelToDb(model) + if len(lastNewModelId) > 0 { + //udpate status + models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST) + } + log.Info("save model end.") return nil @@ -96,9 +103,7 @@ func SaveModel(ctx *context.Context) { } func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - dataActualPath := setting.Bucket + "/" + Model_prefix + - models.AttachmentRelativePath(modelUUID) + - "/" + objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey) if err != nil { @@ -108,25 +113,28 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - var size int64 - prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" + + prefix := objectkey + "/" destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" - for _, modelFile := range modelDbResult { - if modelFile.IsDir { - log.Info("copy dir, continue. dir=" + modelFile.FileName) - continue - } - srcKeyName := prefix + modelFile.ParenDir + modelFile.FileName - log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) - destKeyName := destKeyNamePrefix + modelFile.ParenDir + modelFile.FileName - log.Info("Dest key name=" + destKeyName) - err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) - if err != nil { - log.Info("copy failed.") - } - size += modelFile.Size - } + size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix) + + // for _, modelFile := range modelDbResult { + // if modelFile.IsDir { + // log.Info("copy dir, continue. dir=" + modelFile.FileName) + // continue + // } + // srcKeyName := prefix + modelFile.FileName + // log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) + // destKeyName := destKeyNamePrefix + modelFile.FileName + // log.Info("Dest key name=" + destKeyName) + // err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) + // if err != nil { + // log.Info("copy failed.") + // } + // size += modelFile.Size + // } + dataActualPath := setting.Bucket + "/" + destKeyNamePrefix return dataActualPath, size, nil } @@ -156,8 +164,15 @@ func DeleteModelByID(id string) error { return err } } - - return models.DeleteModelById(id) + err = models.DeleteModelById(id) + if err == nil { //find a model to change new + if model.New == MODEL_LATEST { + aimodels := models.QueryModelByName(model.Name, model.RepoId) + if len(aimodels) > 0 { + models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST) + } + } + } } return err } @@ -175,6 +190,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in }, RepoID: repoId, Type: -1, + New: MODEL_LATEST, }) } @@ -221,6 +237,17 @@ func ShowSingleModel(ctx *context.Context) { ctx.HTML(200, tplModelManageDownload) } +func ShowOneVersionOtherModel(ctx *context.Context) { + repoId := ctx.Repo.Repository.ID + name := ctx.Query("name") + aimodels := models.QueryModelByName(name, repoId) + if len(aimodels) > 0 { + ctx.JSON(200, aimodels[1:]) + } else { + ctx.JSON(200, aimodels) + } +} + func ShowModelPageInfo(ctx *context.Context) { log.Info("ShowModelInfo start.") @@ -237,6 +264,7 @@ func ShowModelPageInfo(ctx *context.Context) { }, RepoID: repoId, Type: Type, + New: MODEL_LATEST, }) if err != nil { ctx.ServerError("Cloudbrain", err) @@ -250,7 +278,6 @@ func ShowModelPageInfo(ctx *context.Context) { ctx.Data["Tasks"] = modelResult ctx.HTML(200, tplModelManageIndex) - } func ModifyModel(id string, description string) error { From dc4fa1fb9efabeae1c90571e63b2399a18381fff Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 11:51:27 +0800 Subject: [PATCH 050/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 3b85a0b20..ad6f52aef 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -214,6 +214,7 @@ func DownloadSingleModelFile(ctx *context.Context) { func ShowSingleModel(ctx *context.Context) { id := ctx.Params(":ID") + parentDir := ctx.Query("parentDir") log.Info("Show single ModelInfo start.id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -222,7 +223,7 @@ func ShowSingleModel(ctx *context.Context) { return } log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) - models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:]) + models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) From d596b5228b8f3c72980ba6a864749d99adf2ab9a Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 14:46:49 +0800 Subject: [PATCH 051/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index a0fa56407..4b5edaa55 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -335,6 +335,7 @@ func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir s fileName = str1[len(str1)-1] isDir = false } + log.Info("fileName=" + fileName) fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, From 56dc817cc0e738652cb100e006ded3f5870ea43d Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 14:48:36 +0800 Subject: [PATCH 052/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 4b5edaa55..54d4a0a21 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -322,6 +322,7 @@ func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir s if err == nil { for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") + log.Info("val.Key=" + val.Key) var isDir bool var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { From 4599b54f97e9268aa71d81f617e41c531754748f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 14:52:00 +0800 Subject: [PATCH 053/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 54d4a0a21..f22bc1dda 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -236,10 +236,10 @@ func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPa log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { - if strings.HasSuffix(val.Key, "/") { - log.Info("copy file, src key=" + val.Key + " is dir, not copy.") - continue - } + // if strings.HasSuffix(val.Key, "/") { + // log.Info("copy file, src key=" + val.Key + " is dir, not copy.") + // continue + // } destKey := destPath + val.Key[length:] log.Info("copy file,bucket=" + srcBucket + " src key=" + val.Key + "destbucket=" + destBucket + " dest key=" + destKey) ObsCopyFile(srcBucket, val.Key, destBucket, destKey) From 6598821e190f56b93fe2388f913c9d6013cd6307 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:04:25 +0800 Subject: [PATCH 054/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 10 +++++++++- routers/repo/ai_model_manage.go | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index f22bc1dda..115fbdc45 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -286,9 +286,17 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro input.Prefix = prefix output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) + prefixLen := len(prefix) + 1 if err == nil { for _, val := range output.Contents { var isDir bool + if val.Key == prefix { + continue + } + log.Info("sub str=" + val.Key[prefixLen:]) + if strings.Contains(val.Key[prefixLen:], "/") { + continue + } if strings.HasSuffix(val.Key, "/") { isDir = true } else { @@ -296,7 +304,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro } fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), - FileName: val.Key[len(prefix)+1:], + FileName: val.Key[prefixLen:], Size: val.Size, IsDir: isDir, ParenDir: "", diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ad6f52aef..949c33bb3 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -214,7 +214,7 @@ func DownloadSingleModelFile(ctx *context.Context) { func ShowSingleModel(ctx *context.Context) { id := ctx.Params(":ID") - parentDir := ctx.Query("parentDir") + //parentDir := ctx.Query("parentDir") log.Info("Show single ModelInfo start.id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -223,7 +223,7 @@ func ShowSingleModel(ctx *context.Context) { return } log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) - models, err := storage.GetObsListObjectByBucketAndPrefix(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir) + models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:]) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) From 13e3634f22122619b77b19a414c80d0d58f4fa12 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:05:58 +0800 Subject: [PATCH 055/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 115fbdc45..d1bbb626f 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -286,7 +286,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro input.Prefix = prefix output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) - prefixLen := len(prefix) + 1 + prefixLen := len(prefix) if err == nil { for _, val := range output.Contents { var isDir bool From d071f15a47768a1b0a5bd1e76ce5f76df76978b3 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:08:57 +0800 Subject: [PATCH 056/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index d1bbb626f..9137d5c1d 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -293,8 +293,8 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro if val.Key == prefix { continue } - log.Info("sub str=" + val.Key[prefixLen:]) - if strings.Contains(val.Key[prefixLen:], "/") { + log.Info("sub str=" + val.Key[prefixLen:len(val.Key)-1]) + if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { continue } if strings.HasSuffix(val.Key, "/") { From 969a69a10b3cc549811036db380d46553ab3fdca Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:27:21 +0800 Subject: [PATCH 057/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 9 ++++++--- routers/repo/ai_model_manage.go | 13 ++++++++----- routers/routes/routes.go | 1 + templates/repo/modelmanage/download.tmpl | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 9137d5c1d..f7f91dcd2 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -290,24 +290,27 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro if err == nil { for _, val := range output.Contents { var isDir bool + var fileName, parentDir string if val.Key == prefix { continue } - log.Info("sub str=" + val.Key[prefixLen:len(val.Key)-1]) if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { continue } if strings.HasSuffix(val.Key, "/") { isDir = true + fileName = val.Key[prefixLen : len(val.Key)-1] + parentDir = val.Key[prefixLen:] } else { isDir = false + fileName = val.Key[prefixLen:] } fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), - FileName: val.Key[prefixLen:], + FileName: fileName, Size: val.Size, IsDir: isDir, - ParenDir: "", + ParenDir: parentDir, } fileInfos = append(fileInfos, fileInfo) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 949c33bb3..eafd98e85 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -202,8 +202,11 @@ func DownloadMultiModelFile(ctx *context.Context) { func DownloadSingleModelFile(ctx *context.Context) { log.Info("DownloadSingleModelFile start.") - path := ctx.Query("path") - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path[len(setting.Bucket)+1:]) + id := ctx.Params(":ID") + parentDir := ctx.Query("parentDir") + fileName := ctx.Query("fileName") + path := Model_prefix + models.AttachmentRelativePath(id) + "/" + parentDir + fileName + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) if err != nil { log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("GetObsCreateSignedUrl", err) @@ -214,7 +217,7 @@ func DownloadSingleModelFile(ctx *context.Context) { func ShowSingleModel(ctx *context.Context) { id := ctx.Params(":ID") - //parentDir := ctx.Query("parentDir") + parentDir := ctx.Query("parentDir") log.Info("Show single ModelInfo start.id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -222,8 +225,8 @@ func ShowSingleModel(ctx *context.Context) { ctx.ServerError("no such model:", err) return } - log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) - models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:]) + log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:] + parentDir) + models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:]+parentDir) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 8c37395a0..aaa2bd6a8 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -964,6 +964,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.ShowModelPageInfo) m.Group("/:ID", func() { m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) + m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile) }) }, context.RepoRef()) diff --git a/templates/repo/modelmanage/download.tmpl b/templates/repo/modelmanage/download.tmpl index 235529859..72e30c1ee 100644 --- a/templates/repo/modelmanage/download.tmpl +++ b/templates/repo/modelmanage/download.tmpl @@ -27,7 +27,7 @@ ' html += `` - html += `` + if(key==="Description"){ + let description = obj[key] + html += '' + // html += '' + }else{ + html += `` + } html += '' } diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index 49fa6a349..c2839fddb 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -103,9 +103,9 @@ From 9ab213bc6acd9dd5bc0843eb6471c259ac505d5b Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 6 Dec 2021 19:23:37 +0800 Subject: [PATCH 174/206] iso --- custom/conf/app.ini.sample | 62 ++++++++++++++++++++++---------------------- modules/setting/setting.go | 64 ++++++++++++++++++++++++---------------------- modules/storage/local.go | 4 +++ modules/storage/minio.go | 6 +++++ modules/storage/storage.go | 1 + routers/repo/cloudbrain.go | 35 +++++++++++++++++++++++-- 6 files changed, 108 insertions(+), 64 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 8bc971f2c..d294c8823 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1059,50 +1059,50 @@ RESULT_BACKEND = redis://localhost:6379 [cloudbrain] USERNAME = PASSWORD = -REST_SERVER_HOST = http://192.168.202.73 -JOB_PATH = /datasets/minio/data/opendata/jobs/ -DEBUG_SERVER_HOST = http://192.168.202.73/ +REST_SERVER_HOST = +JOB_PATH = +DEBUG_SERVER_HOST = ; cloudbrain visit opendata -USER = cW4cMtH24eoWPE7X -PWD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C -GPU_TYPE_DEFAULT = openidebug -GPU_TYPES = {"gpu_type":[{"id":1,"queue":"openidebug","value":"T4"},{"id":2,"queue":"openidgx","value":"V100"}]} +USER = +PWD = +GPU_TYPE_DEFAULT = +GPU_TYPES = [benchmark] -ENABLED = true -BENCHMARKCODE = https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git -HOST = http://192.168.202.90:3366/ +ENABLED = +BENCHMARKCODE = +HOST = [snn4imagenet] -ENABLED = true -SNN4IMAGENETCODE = https://yult:eh2Ten4iLYjFkbj@git.openi.org.cn/ylt/snn4imagenet.git -HOST = http://192.168.202.90:3366/ +ENABLED = +SNN4IMAGENETCODE = +HOST = [decompress] -HOST = http://192.168.207.34:39987 -USER = cW4cMtH24eoWPE7X -PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC +HOST = +USER = +PASSWORD = [blockchain] -HOST = http://192.168.207.84:3002/ -COMMIT_VALID_DATE = 2021-01-15 +HOST = +COMMIT_VALID_DATE = [obs] -ENDPOINT = https://obs.cn-south-222.ai.pcl.cn -ACCESS_KEY_ID = FDP3LRMHLB9S77VWEHE3 -SECRET_ACCESS_KEY = LyM82Wk80pgjhs2z7AdDcsdpCWhbsJtSzQ7hkESN -BUCKET = testopendata -LOCATION = cn-south-222 -BASE_PATH = attachment/ +ENDPOINT = +ACCESS_KEY_ID = +SECRET_ACCESS_KEY = +BUCKET = +LOCATION = +BASE_PATH = [modelarts] -ORGANIZATION = modelarts -ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn -PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa -PROJECT_NAME = cn-south-222_test -USERNAME = test1 -PASSWORD = Qizhi@test. -DOMAIN = cn-south-222 +ORGANIZATION = +ENDPOINT = +PROJECT_ID = +PROJECT_NAME = +USERNAME = +PASSWORD = +DOMAIN = [radar_map] impact=0.3 diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 92eae63b0..3f7345900 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -439,14 +439,15 @@ var ( DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string //benchmark config IsBenchmarkEnabled bool @@ -549,7 +550,7 @@ var ( RecordBeginTime string IgnoreMirrorRepo bool }{} - + Warn_Notify_Mails []string ) @@ -1216,8 +1217,8 @@ func NewContext() { sec = Cfg.Section("decompress") DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987") - AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + AuthUser = sec.Key("USER").MustString("") + AuthPassword = sec.Key("PASSWORD").MustString("") sec = Cfg.Section("labelsystem") LabelTaskName = sec.Key("LabelTaskName").MustString("LabelRedisQueue") @@ -1225,10 +1226,11 @@ func NewContext() { DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") sec = Cfg.Section("cloudbrain") - CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + CBAuthUser = sec.Key("USER").MustString("") + CBAuthPassword = sec.Key("PWD").MustString("") RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73") JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/") + CBCodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("jobs/") DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug") GpuTypes = sec.Key("GPU_TYPES").MustString("") @@ -1236,31 +1238,31 @@ func NewContext() { sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") - BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git") - Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git") - BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") sec = Cfg.Section("obs") - Endpoint = sec.Key("ENDPOINT").MustString("112.95.163.82") + Endpoint = sec.Key("ENDPOINT").MustString("") AccessKeyID = sec.Key("ACCESS_KEY_ID").MustString("") SecretAccessKey = sec.Key("SECRET_ACCESS_KEY").MustString("") - Bucket = sec.Key("BUCKET").MustString("testopendata") - Location = sec.Key("LOCATION").MustString("cn-south-222") - BasePath = sec.Key("BASE_PATH").MustString("attachment/") + Bucket = sec.Key("BUCKET").MustString("") + Location = sec.Key("LOCATION").MustString("") + BasePath = sec.Key("BASE_PATH").MustString("") TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/") OutPutPath = sec.Key("Output_Path").MustString("output/") CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") @@ -1268,17 +1270,17 @@ func NewContext() { PROXYURL = sec.Key("PROXY_URL").MustString("") sec = Cfg.Section("modelarts") - ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") - IamHost = sec.Key("IAMHOST").MustString("112.95.163.80") + ModelArtsHost = sec.Key("ENDPOINT").MustString("") + IamHost = sec.Key("IAMHOST").MustString("") ProjectID = sec.Key("PROJECT_ID").MustString("") ProjectName = sec.Key("PROJECT_NAME").MustString("") ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") - ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") + ModelArtsDomain = sec.Key("DOMAIN").MustString("") AllowedOrg = sec.Key("ORGANIZATION").MustString("") ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") - Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2") + Flavor = sec.Key("FLAVOR").MustString("") ResourcePools = sec.Key("Resource_Pools").MustString("") Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("") @@ -1286,10 +1288,10 @@ func NewContext() { TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") sec = Cfg.Section("elk") - ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") - ElkUser = sec.Key("ELKUSER").MustString("Qizhi") - ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020") - Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") + ElkUrl = sec.Key("ELKURL").MustString("") + ElkUser = sec.Key("ELKUSER").MustString("") + ElkPassword = sec.Key("ELKPASSWORD").MustString("") + Index = sec.Key("INDEX").MustString("") TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") diff --git a/modules/storage/local.go b/modules/storage/local.go index c462dcd9e..d46a5528d 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -76,3 +76,7 @@ func (l *LocalStorage) PresignedPutURL(path string) (string, error) { func (l *LocalStorage) HasObject(path string) (bool, error) { return false, nil } + +func (l *LocalStorage) UploadObject(fileName, filePath string) error { + return nil +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index b14442d56..664e58d1b 100755 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -122,3 +122,9 @@ func (m *MinioStorage) HasObject(path string) (bool, error) { return hasObject, nil } + +//upload object +func (m *MinioStorage) UploadObject(fileName, filePath string) error { + _, err := m.client.FPutObject(m.bucket, fileName, filePath, minio.PutObjectOptions{}) + return err +} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 191871d93..d364346f0 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -26,6 +26,7 @@ type ObjectStorage interface { PresignedGetURL(path string, fileName string) (string, error) PresignedPutURL(path string) (string, error) HasObject(path string) (bool, error) + UploadObject(fileName, filePath string) error } // Copy copys a file from source ObjectStorage to dest ObjectStorage diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b2ded0752..a3af588fb 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -249,6 +249,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } repo := ctx.Repo.Repository downloadCode(repo, codePath) + uploadCodeToMinio(codePath + "/", jobName, "/code/") modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath err = os.MkdirAll(modelPath, os.ModePerm) @@ -267,16 +268,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } } downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) @@ -703,8 +707,8 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc command := "git clone " + gitPath + " " + codePath cmd := exec.Command("/bin/bash", "-c", command) - output, err := cmd.Output() - log.Info(string(output)) + _, err = cmd.Output() + if err != nil { log.Error("exec.Command(%s) failed:%v", command, err) return err @@ -741,6 +745,33 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc return nil } +func uploadCodeToMinio(codePath, jobName, parentDir string) error { + files, err := readDir(codePath) + if err != nil { + log.Error("readDir(%s) failed: %s", codePath, err.Error()) + return err + } + + for _, file := range files { + if file.IsDir() { + if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil { + log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error()) + return err + } + } else { + destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name() + sourceFile := codePath + file.Name() + err = storage.Attachments.UploadObject(destObject, sourceFile) + if err != nil { + log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error()) + return err + } + } + } + + return nil +} + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { From 849d564d72fa9f1beb9ba72241ec21a5b8793a3b Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 08:44:06 +0800 Subject: [PATCH 175/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 76702a8b2..574cdbc5a 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -472,7 +472,7 @@ func ModifyModel(id string, description string) error { } func ModifyModelInfo(ctx *context.Context) { - log.Info("delete model start.") + log.Info("modify model start.") id := ctx.Query("ID") description := ctx.Query("Description") @@ -489,10 +489,11 @@ func ModifyModelInfo(ctx *context.Context) { err = ModifyModel(id, description) - if err == nil { - ctx.HTML(200, "success") + if err != nil { + log.Info("modify error," + err.Error()) + ctx.ServerError("error.", err) } else { - ctx.HTML(500, "Failed.") + ctx.JSON(200, "success") } } From 7a4d8d195b64d3aca89c293de8ba922dc36f78e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 09:43:22 +0800 Subject: [PATCH 176/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/view_list.tmpl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index 3640ff133..1d4a6b811 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -5,14 +5,17 @@ {{if .LatestCommitUser}} {{if .LatestCommitUser.FullName}} - {{.LatestCommitUser.FullName}} + {{.LatestCommitUser.FullName}} + {{.LatestCommitUser.Committer.Name}} {{else}} - {{if .LatestCommit.Author}}{{.LatestCommit.Author.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} + {{if .LatestCommit.Author}}{{.LatestCommit.Author.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} + {{if .LatestCommit.Committer}}{{.LatestCommit.Committer.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} {{end}} {{else}} {{if .LatestCommit.Author}} - {{.LatestCommit.Author.Name}} + {{.LatestCommit.Author.Name}} + {{.LatestCommit.Committer.Name}} {{end}} {{end}} From c2a771ebd0edad82d5b4a3ee2af764dfef2ffae7 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 09:52:05 +0800 Subject: [PATCH 177/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/view_list.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index 1d4a6b811..f668dccee 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -6,10 +6,10 @@ {{if .LatestCommitUser.FullName}} {{.LatestCommitUser.FullName}} - {{.LatestCommitUser.Committer.Name}} + {{.LatestCommit.Committer.Name}} {{else}} {{if .LatestCommit.Author}}{{.LatestCommit.Author.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} - {{if .LatestCommit.Committer}}{{.LatestCommit.Committer.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} + {{.LatestCommit.Committer.Name}} {{end}} {{else}} {{if .LatestCommit.Author}} From 6b11d02663bdd677dc5cff7f65330f3338ff8628 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 09:59:32 +0800 Subject: [PATCH 178/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/hook.go | 3 --- routers/repo/view.go | 26 ++++++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/routers/private/hook.go b/routers/private/hook.go index 59ea68c04..34627a064 100755 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -164,7 +164,6 @@ func isErrUnverifiedCommit(err error) bool { // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { - log.Info("Git pre start..................................") ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) @@ -372,8 +371,6 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { // HookPostReceive updates services and users func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { - log.Info("Git post start..................................") - ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") diff --git a/routers/repo/view.go b/routers/repo/view.go index a5f2e1477..d2b68e74f 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -7,8 +7,8 @@ package repo import ( "bytes" - "code.gitea.io/gitea/modules/options" "encoding/base64" + "encoding/json" "fmt" gotemplate "html/template" "io/ioutil" @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" ) @@ -343,6 +344,11 @@ func renderDirectory(ctx *context.Context, treeLink string) { // Show latest commit info of repository in table header, // or of directory if not in root directory. ctx.Data["LatestCommit"] = latestCommit + + latestCommitJson, _ := json.Marshal(latestCommit) + //log.Info(accuracyJson=” + string(accuracyJson)) + log.Info("latestCommitJson=" + string(latestCommitJson)) + verification := models.ParseCommitWithSignature(latestCommit) if err := models.CalculateTrustStatus(verification, ctx.Repo.Repository, nil); err != nil { @@ -578,11 +584,11 @@ func safeURL(address string) string { } type ContributorInfo struct { - UserInfo *models.User // nil for contributor who is not a registered user - RelAvatarLink string `json:"rel_avatar_link"` - UserName string `json:"user_name"` - Email string `json:"email"` - CommitCnt int `json:"commit_cnt"` + UserInfo *models.User // nil for contributor who is not a registered user + RelAvatarLink string `json:"rel_avatar_link"` + UserName string `json:"user_name"` + Email string `json:"email"` + CommitCnt int `json:"commit_cnt"` } type GetContributorsInfo struct { @@ -641,7 +647,7 @@ func Home(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "",c.Email, c.CommitCnt, + user, "", "", c.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -838,7 +844,7 @@ func renderCode(ctx *context.Context) { compareInfo, err = baseGitRepo.GetCompareInfo(ctx.Repo.Repository.RepoPath(), ctx.Repo.BranchName, ctx.Repo.Repository.BaseRepo.DefaultBranch) ctx.Data["UpstreamSameBranchName"] = false } - if err==nil && compareInfo != nil { + if err == nil && compareInfo != nil { if compareInfo.Commits != nil { log.Info("compareInfoCommits数量:%d", compareInfo.Commits.Len()) ctx.Data["FetchUpstreamCnt"] = compareInfo.Commits.Len() @@ -942,7 +948,7 @@ func ContributorsAPI(ctx *context.Context) { } else { // new committer info var newContributor = &ContributorInfo{ - user, user.RelAvatarLink(),user.Name, user.Email,c.CommitCnt, + user, user.RelAvatarLink(), user.Name, user.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -955,7 +961,7 @@ func ContributorsAPI(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "",c.Email,c.CommitCnt, + user, "", "", c.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) From dc275febc4be85c7f08c26243ead9e6e180234cb Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 10:12:34 +0800 Subject: [PATCH 179/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 574cdbc5a..c497675fb 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -391,6 +391,11 @@ func ShowOneVersionOtherModel(ctx *context.Context) { repoId := ctx.Repo.Repository.ID name := ctx.Query("name") aimodels := models.QueryModelByName(name, repoId) + for _, model := range aimodels { + log.Info("model=" + model.Name) + log.Info("model.UserId=" + fmt.Sprint(model.UserId)) + model.IsCanOper = isOper(ctx, model.UserId) + } if len(aimodels) > 0 { ctx.JSON(200, aimodels[1:]) } else { From b8cc05d25218a4343f40880520b5d366fc372af0 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 7 Dec 2021 10:13:06 +0800 Subject: [PATCH 180/206] fix issue --- templates/repo/debugjob/index.tmpl | 575 +++++++++++++++++++++++++++++++ templates/repo/modelmanage/index.tmpl | 14 +- templates/repo/modelmanage/showinfo.tmpl | 34 +- 3 files changed, 592 insertions(+), 31 deletions(-) create mode 100644 templates/repo/debugjob/index.tmpl diff --git a/templates/repo/debugjob/index.tmpl b/templates/repo/debugjob/index.tmpl new file mode 100644 index 000000000..83a12751d --- /dev/null +++ b/templates/repo/debugjob/index.tmpl @@ -0,0 +1,575 @@ + +{{template "base/head" .}} + + + + +
+
+
+
+
+
+
+
+
+ +
+ +
+ {{template "repo/header" .}} + +
+
+
+
+ {{$.i18n.Tr "repo.modelarts.notebook"}} + {{$.i18n.Tr "repo.modelarts.train_job"}} +
+
+
+
+ {{svg "octicon-server" 16}} +
全部
+ +
+ 全部 + CPU / GPU + Ascend NPU +
+
+ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.modelarts.train_job.new_debug"}}{{end}} +
+
+ +
+
+
+ + +
+
+
+
+ {{$.i18n.Tr "repo.cloudbrain_task"}} +
+
+ {{$.i18n.Tr "repo.modelarts.status"}} +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}} +
+
+ {{$.i18n.Tr "repo.modelarts.computing_resources"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_creator"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_operate"}} +
+
+
+ {{range .Tasks}} +
+
+ +
+ + {{.JobName}} + +
+
+ + + {{.Status}} + +
+
+ + {{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}} +
+
+ + {{.ComputeResource}} +
+
+ {{if .User.Name}} + + {{else}} + + {{end}} +
+
+
+ {{if and (ne .Status "WAITING") (ne .JobType "DEBUG")}} + + 评分 + + {{end}} + + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + {{if eq .ComputeResource "CPU/GPU"}} + + {{$.i18n.Tr "repo.debug"}} + + {{else}} + + {{$.i18n.Tr "repo.debug"}} + + {{end}} + {{else}} + + {{$.i18n.Tr "repo.debug"}} + + {{end}} + +
+ {{$.CsrfTokenHtml}} + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.stop"}} + + {{else}} + + {{$.i18n.Tr "repo.stop"}} + + {{end}} + + +
+
+ + + + + + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.submit_image"}} + {{else}} + {{$.i18n.Tr "repo.submit_image"}} + {{end}} +
+ {{.CanDel}} + +
+ + {{$.CsrfTokenHtml}} + {{if .CanDel}} + + + {{$.i18n.Tr "repo.delete"}} + + {{else}} + + {{$.i18n.Tr "repo.delete"}} + + {{end}} + + +
+ + +
+
+ {{end}} {{template "base/paginate" .}} +
+
+
+
+
+
+ + + + +
+
+
+ 删除任务 +
+ +
+

你确认删除该任务么？此任务一旦删除不可恢复。

+
+
+
+ 取消操作 +
+
+ 确定操作 +
+
+
+
+ + +{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index b9b76cb18..eab46387b 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -93,17 +93,11 @@
- @@ -118,16 +112,12 @@
-
- +
@@ -169,8 +159,6 @@ function createModelName(){ let repoName = location.pathname.split('/')[2] let modelName = repoName + '_model_' + Math.random().toString(36).substr(2, 4) - - console.log("repoName",modelName) $('#name').val(modelName) $('#version').val("0.0.1") } diff --git a/templates/repo/modelmanage/showinfo.tmpl b/templates/repo/modelmanage/showinfo.tmpl index a4ef84ebc..821340eb0 100644 --- a/templates/repo/modelmanage/showinfo.tmpl +++ b/templates/repo/modelmanage/showinfo.tmpl @@ -32,10 +32,17 @@ border-radius: .38571429rem !important; margin-left: 1.5em; } + .title-word-elipsis{ + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + width: 30%; + } .word-elipsis{ overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + padding-right: 80px; } .half-table{ width: 50%; @@ -78,8 +85,6 @@ - \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 5d93292a9..f523d96a9 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -62,7 +62,6 @@
{{$.i18n.Tr "repo.modelarts.notebook"}} {{$.i18n.Tr "repo.modelarts.train_job"}} - 模型管理
diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index eab46387b..6d89fd50b 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -181,31 +181,6 @@ }) .modal('show') } - // 删除时用户确认 - function assertDelete(obj) { - if (obj.style.color == "rgb(204, 204, 204)") { - return - } else { - var delId = obj.parentNode.id - flag = 1; - $('.ui.basic.modal.first') - .modal({ - onDeny: function() { - flag = false - }, - onApprove: function() { - document.getElementById(delId).submit() - flag = true - }, - onHidden: function() { - if (flag == false) { - $('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut(); - } - } - }) - .modal('show') - } - } $(function(){ $('#choice_model').dropdown({ @@ -225,11 +200,11 @@ }else{ versionArray[1]=String(Number(versionArray[1])+1) } - + + versionArray[2]='0' }else{ versionArray[2]=String(Number(versionArray[2])+1) } - console.log(versionArray.join('.')) return versionArray.join('.') } function loadTrainList(){ diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index 14a625c6c..cfc51be0e 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -3,6 +3,7 @@ @@ -134,9 +135,6 @@ const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config; - - - export default { components: { @@ -150,32 +148,30 @@ export default { tableData: [], fullscreenLoading: false, url:'', - isLoading:true + isLoading:true, + loadNodeMap:new Map() }; }, methods: { - load(tree, treeNode, resolve) { - console.log("tree",tree) + load(tree, treeNode, resolve) { + this.loadNodeMap.set(tree.cName, tree.ID) this.$axios.get(this.url+'show_model_child_api',{params:{ - name:tree.Name + name:tree.cName }}).then((res)=>{ - console.log(res) let TrainTaskInfo let tableData tableData= res.data - console.log("-----tableData---",tableData) for(let i=0;i{ - console.log(res,tree) _this.getModelList() - _this.load(tree) + _this.loadrefresh(tree) }) flag = true }, @@ -240,7 +234,26 @@ export default { } }) .modal('show') - + }, + loadrefresh(tree){ + this.$axios.get(this.url+'show_model_child_api',{params:{ + name:tree.cName + }}).then((res)=>{ + let TrainTaskInfo + let tableData + tableData= res.data + for(let i=0;i{ - console.log("res",res) let TrainTaskInfo this.tableData = res.data.data - console.log("this.tableData",this.tableData) for(let i=0;i Date: Tue, 7 Dec 2021 15:09:23 +0800 Subject: [PATCH 187/206] fix-1033 --- modules/normalization/normalization.go | 11 ++++++++--- modules/setting/setting.go | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/normalization/normalization.go b/modules/normalization/normalization.go index ce616d7f8..a258a13a7 100644 --- a/modules/normalization/normalization.go +++ b/modules/normalization/normalization.go @@ -4,6 +4,8 @@ import ( "code.gitea.io/gitea/modules/setting" ) +const MAX_LINES_RECORD = 100 + func Normalization(value float64, minValue float64, maxValue float64) float64 { min := int64(minValue * 100) @@ -72,9 +74,12 @@ func GetTeamHealthInitValue(contributors int64, keyContributors int64, newContri } -func GetRepoGrowthInitValue(codelinesGrowth int64, issueGrowth int64, commitsGrowth int64, newContributors int64, commentsGrowth int64) float64 { - - return setting.RadarMap.GrowthCodeLines*float64(codelinesGrowth) + +func GetRepoGrowthInitValue(codeLinesGrowth int64, issueGrowth int64, commitsGrowth int64, newContributors int64, commentsGrowth int64) float64 { + codeLinesKB := codeLinesGrowth / 1000 + if codeLinesKB > MAX_LINES_RECORD { + codeLinesKB = MAX_LINES_RECORD + } + return setting.RadarMap.GrowthCodeLines*float64(codeLinesKB) + setting.RadarMap.GrowthIssue*float64(issueGrowth) + setting.RadarMap.GrowthCommit*float64(commitsGrowth) + setting.RadarMap.GrowthContributors*float64(newContributors) + diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 92eae63b0..ec130ff4e 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1313,7 +1313,7 @@ func SetRadarMapConfig() { RadarMap.CompletenessIssuesClosed = sec.Key("completeness_issues_closed").MustFloat64(0.2) RadarMap.CompletenessReleases = sec.Key("completeness_releases").MustFloat64(0.3) RadarMap.CompletenessDevelopAge = sec.Key("completeness_develop_age").MustFloat64(0.1) - RadarMap.CompletenessDataset = sec.Key("completeness_dataset").MustFloat64(0.1) + RadarMap.CompletenessDataset = sec.Key("completeness_dataset").MustFloat64(0) RadarMap.CompletenessModel = sec.Key("completeness_model").MustFloat64(0.1) RadarMap.CompletenessWiki = sec.Key("completeness_wiki").MustFloat64(0.1) RadarMap.Liveness = sec.Key("liveness").MustFloat64(0.3) From df5f3010edbab1eeb49c2eb24672bd3f520f7dcf Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 15:12:59 +0800 Subject: [PATCH 188/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 8ce8dfa15..afe45c942 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -412,6 +412,7 @@ func ShowOneVersionOtherModel(ctx *context.Context) { } func ShowModelTemplate(ctx *context.Context) { + ctx.Data["isModelManage"] = true ctx.HTML(200, tplModelManageIndex) } From 7c1a008b598c982ae49b278a489a0f87f1abe2af Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 7 Dec 2021 15:13:41 +0800 Subject: [PATCH 189/206] fix issue --- templates/repo/header.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 5ee05b080..0b45d9b67 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -138,7 +138,7 @@ {{end}} {{if .Permission.CanRead $.UnitTypeDatasets}} - + {{.i18n.Tr "repo.model_manager"}} From 7d12704c737b8e1d01138b91cfb61b3d4a24925f Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 7 Dec 2021 15:20:25 +0800 Subject: [PATCH 190/206] fix issue --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + templates/repo/settings/options.tmpl | 9 ++++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 311191e5e..3ab3a00c9 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1575,6 +1575,7 @@ settings.external_wiki_url_error = The external wiki URL is not a valid URL. settings.external_wiki_url_desc = Visitors are redirected to the external wiki URL when clicking the wiki tab. settings.dataset_desc = Enable Repository Dataset settings.cloudbrain_desc = Enable Cloudbarin +settings.model_desc = Enable Model Manage settings.issues_desc = Enable Repository Issue Tracker settings.use_internal_issue_tracker = Use Built-In Issue Tracker settings.use_external_issue_tracker = Use External Issue Tracker diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 94b06ac14..8bd04459a 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1586,6 +1586,7 @@ settings.external_wiki_url_error=外部百科链接无效 settings.external_wiki_url_desc=当点击任务标签时，访问者将被重定向到外部任务系统的URL。 settings.dataset_desc=启用数据集 settings.cloudbrain_desc = 启用云脑 +settings.model_desc = 启用模型管理 settings.issues_desc=启用任务系统 settings.use_internal_issue_tracker=使用内置的轻量级任务管理系统 settings.use_external_issue_tracker=使用外部的任务管理系统 diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index ed30e97a9..8c0e53a29 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -158,7 +158,14 @@
- + {{$isCloudBrainEnabled := .Repository.UnitEnabled $.UnitTypeCloudBrain }} +
+ +
+ + +
+
{{$isWikiEnabled := or (.Repository.UnitEnabled $.UnitTypeWiki) (.Repository.UnitEnabled $.UnitTypeExternalWiki)}}
From ea47c6a18cd201081b48f6b63d95feec9a3ab12e Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 15:51:29 +0800 Subject: [PATCH 191/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E7=95=8C=E9=9D=A2Tab?= =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E6=8E=A7=E5=88=B6=E7=9A=84=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/repo.go | 6 ++++++ models/repo_unit.go | 16 ++++++++++++++++ models/unit.go | 14 ++++++++++++++ modules/auth/repo_form.go | 1 + modules/context/repo.go | 1 + routers/repo/ai_model_manage.go | 4 ++-- routers/repo/setting.go | 12 ++++++++++++ routers/routes/routes.go | 20 +++++++++++--------- 8 files changed, 63 insertions(+), 11 deletions(-) diff --git a/models/repo.go b/models/repo.go index 6d73aa28a..7948346c5 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1114,6 +1114,12 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository) (err error Type: tp, Config: &BlockChainConfig{EnableBlockChain: true}, }) + } else if tp == UnitTypeModelManage { + units = append(units, RepoUnit{ + RepoID: repo.ID, + Type: tp, + Config: &ModelManageConfig{EnableModelManage: true}, + }) } else { units = append(units, RepoUnit{ RepoID: repo.ID, diff --git a/models/repo_unit.go b/models/repo_unit.go index 518c4b979..5f118029f 100755 --- a/models/repo_unit.go +++ b/models/repo_unit.go @@ -131,6 +131,20 @@ type CloudBrainConfig struct { EnableCloudBrain bool } +type ModelManageConfig struct { + EnableModelManage bool +} + +// FromDB fills up a CloudBrainConfig from serialized format. +func (cfg *ModelManageConfig) FromDB(bs []byte) error { + return json.Unmarshal(bs, &cfg) +} + +// ToDB exports a CloudBrainConfig to a serialized format. +func (cfg *ModelManageConfig) ToDB() ([]byte, error) { + return json.Marshal(cfg) +} + // FromDB fills up a CloudBrainConfig from serialized format. func (cfg *CloudBrainConfig) FromDB(bs []byte) error { return json.Unmarshal(bs, &cfg) @@ -176,6 +190,8 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { r.Config = new(CloudBrainConfig) case UnitTypeBlockChain: r.Config = new(BlockChainConfig) + case UnitTypeModelManage: + r.Config = new(ModelManageConfig) default: panic("unrecognized repo unit type: " + com.ToStr(*val)) } diff --git a/models/unit.go b/models/unit.go index e2b73841a..381491388 100755 --- a/models/unit.go +++ b/models/unit.go @@ -27,6 +27,7 @@ const ( UnitTypeDatasets UnitType = 10 // 10 Dataset UnitTypeCloudBrain UnitType = 11 // 11 CloudBrain UnitTypeBlockChain UnitType = 12 // 12 BlockChain + UnitTypeModelManage UnitType = 13 // 13 ModelManage ) // Value returns integer value for unit type @@ -56,6 +57,8 @@ func (u UnitType) String() string { return "UnitTypeCloudBrain" case UnitTypeBlockChain: return "UnitTypeBlockChain" + case UnitTypeModelManage: + return "UnitTypeModelManage" } return fmt.Sprintf("Unknown UnitType %d", u) } @@ -80,6 +83,7 @@ var ( UnitTypeDatasets, UnitTypeCloudBrain, UnitTypeBlockChain, + UnitTypeModelManage, } // DefaultRepoUnits contains the default unit types @@ -92,6 +96,7 @@ var ( UnitTypeDatasets, UnitTypeCloudBrain, UnitTypeBlockChain, + UnitTypeModelManage, } // NotAllowedDefaultRepoUnits contains units that can't be default @@ -281,6 +286,14 @@ var ( 7, } + UnitModelManage = Unit{ + UnitTypeModelManage, + "repo.modelmanage", + "/modelmanage", + "repo.modelmanage.desc", + 8, + } + // Units contains all the units Units = map[UnitType]Unit{ UnitTypeCode: UnitCode, @@ -293,6 +306,7 @@ var ( UnitTypeDatasets: UnitDataset, UnitTypeCloudBrain: UnitCloudBrain, UnitTypeBlockChain: UnitBlockChain, + UnitTypeModelManage: UnitModelManage, } ) diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index 8352026fe..8061c6469 100755 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -122,6 +122,7 @@ type RepoSettingForm struct { // Advanced settings EnableDataset bool EnableCloudBrain bool + EnableModelManager bool EnableWiki bool EnableExternalWiki bool ExternalWikiURL string diff --git a/modules/context/repo.go b/modules/context/repo.go index 9f8a178fc..de494c1bc 100755 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -821,5 +821,6 @@ func UnitTypes() macaron.Handler { ctx.Data["UnitTypeExternalWiki"] = models.UnitTypeExternalWiki ctx.Data["UnitTypeExternalTracker"] = models.UnitTypeExternalTracker ctx.Data["UnitTypeBlockChain"] = models.UnitTypeBlockChain + ctx.Data["UnitTypeModelManage"] = models.UnitTypeModelManage } } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index afe45c942..c6ec6c6ae 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -118,7 +118,7 @@ func SaveModel(ctx *context.Context) { label := ctx.Query("Label") description := ctx.Query("Description") - if !ctx.Repo.CanWrite(models.UnitTypeCloudBrain) { + if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright"))) return } @@ -418,7 +418,7 @@ func ShowModelTemplate(ctx *context.Context) { func isQueryRight(ctx *context.Context) bool { if ctx.Repo.Repository.IsPrivate { - if ctx.Repo.CanRead(models.UnitTypeCloudBrain) || ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() { + if ctx.Repo.CanRead(models.UnitTypeModelManage) || ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() { return true } return false diff --git a/routers/repo/setting.go b/routers/repo/setting.go index bf11f9e5a..f7da8f4a8 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -239,6 +239,18 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeCloudBrain) } + if form.EnableModelManager && !models.UnitTypeModelManage.UnitGlobalDisabled() { + units = append(units, models.RepoUnit{ + RepoID: repo.ID, + Type: models.UnitTypeModelManage, + Config: &models.ModelManageConfig{ + EnableModelManage: form.EnableModelManager, + }, + }) + } else if !models.UnitTypeModelManage.UnitGlobalDisabled() { + deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeModelManage) + } + if form.EnableWiki && form.EnableExternalWiki && !models.UnitTypeExternalWiki.UnitGlobalDisabled() { if !validation.IsValidExternalURL(form.ExternalWikiURL) { ctx.Flash.Error(ctx.Tr("repo.settings.external_wiki_url_error")) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 1866f0db9..a18fe422c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -616,6 +616,8 @@ func RegisterRoutes(m *macaron.Macaron) { reqRepoDatasetWriter := context.RequireRepoWriter(models.UnitTypeDatasets) reqRepoCloudBrainReader := context.RequireRepoReader(models.UnitTypeCloudBrain) reqRepoCloudBrainWriter := context.RequireRepoWriter(models.UnitTypeCloudBrain) + reqRepoModelManageReader := context.RequireRepoReader(models.UnitTypeModelManage) + reqRepoModelManageWriter := context.RequireRepoWriter(models.UnitTypeModelManage) //reqRepoBlockChainReader := context.RequireRepoReader(models.UnitTypeBlockChain) //reqRepoBlockChainWriter := context.RequireRepoWriter(models.UnitTypeBlockChain) @@ -973,21 +975,21 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) m.Group("/modelmanage", func() { - m.Post("/create_model", repo.SaveModel) + m.Post("/create_model", reqRepoModelManageWriter, repo.SaveModel) m.Delete("/delete_model", repo.DeleteModel) m.Put("/modify_model", repo.ModifyModelInfo) - m.Get("/show_model", reqRepoCloudBrainReader, repo.ShowModelTemplate) - m.Get("/show_model_info", reqRepoCloudBrainReader, repo.ShowModelInfo) - m.Get("/show_model_info_api", reqRepoCloudBrainReader, repo.ShowSingleModel) - m.Get("/show_model_api", reqRepoCloudBrainReader, repo.ShowModelPageInfo) - m.Get("/show_model_child_api", reqRepoCloudBrainReader, repo.ShowOneVersionOtherModel) + m.Get("/show_model", reqRepoModelManageReader, repo.ShowModelTemplate) + m.Get("/show_model_info", repo.ShowModelInfo) + m.Get("/show_model_info_api", repo.ShowSingleModel) + m.Get("/show_model_api", repo.ShowModelPageInfo) + m.Get("/show_model_child_api", repo.ShowOneVersionOtherModel) m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList) m.Get("/query_train_job_version", reqRepoCloudBrainReader, repo.QueryTrainJobVersionList) m.Group("/:ID", func() { - m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) - m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile) + m.Get("", repo.ShowSingleModel) + m.Get("/downloadsingle", repo.DownloadSingleModelFile) }) - m.Get("/downloadall", reqRepoCloudBrainReader, repo.DownloadMultiModelFile) + m.Get("/downloadall", repo.DownloadMultiModelFile) }, context.RepoRef()) m.Group("/modelarts", func() { From 4e7b2dd50bbfafa4766b5ba8dd4caae8e995e98c Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 7 Dec 2021 16:50:34 +0800 Subject: [PATCH 192/206] fix issue --- templates/repo/header.tmpl | 2 +- templates/repo/modelmanage/index.tmpl | 14 +++++++++----- templates/repo/settings/options.tmpl | 4 ++-- web_src/js/components/Model.vue | 1 + 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 0b45d9b67..b84f4771f 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -137,7 +137,7 @@ {{svg "octicon-inbox" 16}} {{.i18n.Tr "datasets"}} {{end}} - {{if .Permission.CanRead $.UnitTypeDatasets}} + {{if .Permission.CanRead $.UnitTypeModelManage}} {{.i18n.Tr "repo.model_manager"}} diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 6d89fd50b..8dfed110b 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -28,7 +28,7 @@
{{template "repo/header" .}} -
+
{{template "base/alert" .}}
@@ -87,7 +87,7 @@
-

+
@@ -117,7 +117,7 @@
- +
@@ -177,6 +177,8 @@ $('#choice_model').dropdown('clear') $('#choice_version').dropdown('clear') $('.ui.dimmer').css({"background-color":""}) + $('.ui.error.message').css('display','none') + } }) .modal('show') @@ -200,7 +202,6 @@ }else{ versionArray[1]=String(Number(versionArray[1])+1) } - versionArray[2]='0' }else{ versionArray[2]=String(Number(versionArray[2])+1) @@ -264,6 +265,7 @@ return true } + $('#submitId').click(function(){ let flag=check() if(flag){ @@ -277,10 +279,12 @@ $('.ui.modal.second').modal('hide') window.location.href=location.href }, - error: function (xhr) { + error: function(xhr){ // 隐藏 loading // 只有请求不正常（状态码不为200）才会执行 + console.log("-------------",xhr) $('.ui.error.message').text(xhr.responseText) + $('.ui.error.message').css('display','block') }, complete:function(xhr){ $("#mask").css({"display":"none","z-index":"1"}) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 8c0e53a29..de31aa59b 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -158,11 +158,11 @@
- {{$isCloudBrainEnabled := .Repository.UnitEnabled $.UnitTypeCloudBrain }} + {{$isModelMangeEnabled := .Repository.UnitEnabled $.UnitTypeModelManage }}
- +
diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index cfc51be0e..dafcce29f 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -261,6 +261,7 @@ export default { this.$axios.get(location.href+'_api',{ params:this.params }).then((res)=>{ + $("#loadContainer").removeClass("loader") let TrainTaskInfo this.tableData = res.data.data for(let i=0;i Date: Tue, 7 Dec 2021 16:57:49 +0800 Subject: [PATCH 193/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/hook.go | 1 - routers/private/serv.go | 1 + routers/repo/view.go | 26 ++++++++++---------------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/routers/private/hook.go b/routers/private/hook.go index 34627a064..a673362fa 100755 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -370,7 +370,6 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { // HookPostReceive updates services and users func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { - ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") diff --git a/routers/private/serv.go b/routers/private/serv.go index b76199481..f405d1d75 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -125,6 +125,7 @@ func ServCommand(ctx *macaron.Context) { return } } + for _, verb := range ctx.QueryStrings("verb") { // clone_cnt if verb == "git-upload-pack" { go repo.IncreaseCloneCnt() diff --git a/routers/repo/view.go b/routers/repo/view.go index d2b68e74f..a5f2e1477 100755 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -7,8 +7,8 @@ package repo import ( "bytes" + "code.gitea.io/gitea/modules/options" "encoding/base64" - "encoding/json" "fmt" gotemplate "html/template" "io/ioutil" @@ -29,7 +29,6 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" - "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" ) @@ -344,11 +343,6 @@ func renderDirectory(ctx *context.Context, treeLink string) { // Show latest commit info of repository in table header, // or of directory if not in root directory. ctx.Data["LatestCommit"] = latestCommit - - latestCommitJson, _ := json.Marshal(latestCommit) - //log.Info(accuracyJson=” + string(accuracyJson)) - log.Info("latestCommitJson=" + string(latestCommitJson)) - verification := models.ParseCommitWithSignature(latestCommit) if err := models.CalculateTrustStatus(verification, ctx.Repo.Repository, nil); err != nil { @@ -584,11 +578,11 @@ func safeURL(address string) string { } type ContributorInfo struct { - UserInfo *models.User // nil for contributor who is not a registered user - RelAvatarLink string `json:"rel_avatar_link"` - UserName string `json:"user_name"` - Email string `json:"email"` - CommitCnt int `json:"commit_cnt"` + UserInfo *models.User // nil for contributor who is not a registered user + RelAvatarLink string `json:"rel_avatar_link"` + UserName string `json:"user_name"` + Email string `json:"email"` + CommitCnt int `json:"commit_cnt"` } type GetContributorsInfo struct { @@ -647,7 +641,7 @@ func Home(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "", c.Email, c.CommitCnt, + user, "", "",c.Email, c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -844,7 +838,7 @@ func renderCode(ctx *context.Context) { compareInfo, err = baseGitRepo.GetCompareInfo(ctx.Repo.Repository.RepoPath(), ctx.Repo.BranchName, ctx.Repo.Repository.BaseRepo.DefaultBranch) ctx.Data["UpstreamSameBranchName"] = false } - if err == nil && compareInfo != nil { + if err==nil && compareInfo != nil { if compareInfo.Commits != nil { log.Info("compareInfoCommits数量:%d", compareInfo.Commits.Len()) ctx.Data["FetchUpstreamCnt"] = compareInfo.Commits.Len() @@ -948,7 +942,7 @@ func ContributorsAPI(ctx *context.Context) { } else { // new committer info var newContributor = &ContributorInfo{ - user, user.RelAvatarLink(), user.Name, user.Email, c.CommitCnt, + user, user.RelAvatarLink(),user.Name, user.Email,c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) @@ -961,7 +955,7 @@ func ContributorsAPI(ctx *context.Context) { existedContributorInfo.CommitCnt += c.CommitCnt } else { var newContributor = &ContributorInfo{ - user, "", "", c.Email, c.CommitCnt, + user, "", "",c.Email,c.CommitCnt, } count++ contributorInfos = append(contributorInfos, newContributor) From 5e45cef68651cf8aba5f134445b518ee8520f805 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 17:10:27 +0800 Subject: [PATCH 194/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 19 ---------- routers/home.go | 35 ------------------- routers/repo/http.go | 5 --- routers/routes/routes.go | 2 -- templates/repo/modelmanage/download.tmpl | 59 -------------------------------- templates/repo/view_list.tmpl | 9 ++--- 6 files changed, 3 insertions(+), 126 deletions(-) delete mode 100644 templates/repo/modelmanage/download.tmpl diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 8c6ccfa06..367ffe1e8 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -484,26 +484,7 @@ func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { } func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { - // input := &obs.CreateSignedUrlInput{} - // input.Bucket = setting.Bucket - // input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") - return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")) - - // input.Expires = 60 * 60 - // input.Method = obs.HttpMethodGet - - // reqParams := make(map[string]string) - // reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" - // input.QueryParams = reqParams - // output, err := ObsCli.CreateSignedUrl(input) - // if err != nil { - // log.Error("CreateSignedUrl failed:", err.Error()) - // return "", err - // } - - // return output.SignedUrl, nil - } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { diff --git a/routers/home.go b/routers/home.go index 99472cddb..d2c93c771 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,8 +7,6 @@ package routers import ( "bytes" - "fmt" - "io/ioutil" "net/http" "strings" @@ -513,36 +511,3 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } - -func RecommendOrgFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "organizations" - recommendFromPromote(ctx, url) -} - -func recommendFromPromote(ctx *context.Context, url string) { - resp, err := http.Get(url) - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - bytes, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return - } - - allLineStr := string(bytes) - lines := strings.Split(allLineStr, "\n") - for i, line := range lines { - log.Info("i=" + fmt.Sprint(i) + " line=" + line) - } - ctx.JSON(http.StatusOK, lines) -} - -func RecommendRepoFromPromote(ctx *context.Context) { - url := setting.RecommentRepoAddr + "projects" - recommendFromPromote(ctx, url) -} diff --git a/routers/repo/http.go b/routers/repo/http.go index fe9279a62..87406a2c3 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -9,7 +9,6 @@ import ( "bytes" "compress/gzip" gocontext "context" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -258,7 +257,6 @@ func HTTP(ctx *context.Context) { models.EnvPusherID + fmt.Sprintf("=%d", authUser.ID), models.EnvIsDeployKey + "=false", } - log.Info("username=" + username + ", EnvPusherName=" + authUser.Name + " EnvRepoName=" + reponame) if !authUser.KeepEmailPrivate { environ = append(environ, models.EnvPusherEmail+"="+authUser.Email) } @@ -561,9 +559,6 @@ func serviceRPC(h serviceHandler, service string) { cmd.Env = append(os.Environ(), h.environ...) } - envJson, _ := json.Marshal(cmd.Env) - //log.Info(accuracyJson=” + string(accuracyJson)) - log.Info("env=" + string(envJson)) cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a18fe422c..a8f820dba 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -315,8 +315,6 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) - m.Get("/recommend/org", routers.RecommendOrgFromPromote) - m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") diff --git a/templates/repo/modelmanage/download.tmpl b/templates/repo/modelmanage/download.tmpl deleted file mode 100644 index e36b0e0b5..000000000 --- a/templates/repo/modelmanage/download.tmpl +++ /dev/null @@ -1,59 +0,0 @@ -{{template "base/head" .}} -
- {{template "repo/header" .}} - -
-
-
-

- {{ range $index, $item := .Path }}{{ $item }}/{{ end }} -

-
-
-
- -
-
-
-
- - 下载所有 - - {{if .Dirs}} -
- + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} From 2913e59deb353ff92bbf00ab49b69ee151185349 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:36:28 +0800 Subject: [PATCH 058/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 14 +++++++------- routers/repo/ai_model_manage.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index f7f91dcd2..86d022cee 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -280,18 +280,18 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } -func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, error) { +func GetAllObsListObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket - input.Prefix = prefix + input.Prefix = prefixRootPath + relativePath output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) - prefixLen := len(prefix) + prefixLen := len(input.Prefix) if err == nil { for _, val := range output.Contents { var isDir bool - var fileName, parentDir string - if val.Key == prefix { + var fileName string + if val.Key == input.Prefix { continue } if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { @@ -300,7 +300,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro if strings.HasSuffix(val.Key, "/") { isDir = true fileName = val.Key[prefixLen : len(val.Key)-1] - parentDir = val.Key[prefixLen:] + relativePath += val.Key[prefixLen:] } else { isDir = false fileName = val.Key[prefixLen:] @@ -310,7 +310,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro FileName: fileName, Size: val.Size, IsDir: isDir, - ParenDir: parentDir, + ParenDir: relativePath, } fileInfos = append(fileInfos, fileInfo) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index eafd98e85..bc276ad17 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -225,8 +225,8 @@ func ShowSingleModel(ctx *context.Context) { ctx.ServerError("no such model:", err) return } - log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:] + parentDir) - models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:]+parentDir) + log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) + models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) From fd9a62f7ab422ad3e50cb130fe0e6f8f1ba480f7 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 15:38:08 +0800 Subject: [PATCH 059/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index bc276ad17..da1bc30ac 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -105,7 +105,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") - modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey) + modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey, "") if err != nil { log.Info("get TrainJobListModel failed:", err) return "", 0, err From a787c484f7b597407895b9c85f4c56b81111471d Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 16:51:35 +0800 Subject: [PATCH 060/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 69 ++++++++++--------- routers/repo/ai_model_manage.go | 112 +++++++++++++++++++++++-------- routers/routes/routes.go | 1 + templates/repo/modelmanage/download.tmpl | 2 +- 4 files changed, 126 insertions(+), 58 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 86d022cee..eea716725 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -280,7 +280,7 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } -func GetAllObsListObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { +func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefixRootPath + relativePath @@ -324,47 +324,56 @@ func GetAllObsListObjectUnderDir(bucket string, prefixRootPath string, relativeP } -func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir string) ([]FileInfo, error) { +func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket + // 设置每页100个对象 + input.MaxKeys = 100 input.Prefix = prefix - output, err := ObsCli.ListObjects(input) + index := 1 fileInfos := make([]FileInfo, 0) - if err == nil { - for _, val := range output.Contents { - str1 := strings.Split(val.Key, "/") - log.Info("val.Key=" + val.Key) - var isDir bool - var fileName, nextParentDir string - if strings.HasSuffix(val.Key, "/") { - fileName = str1[len(str1)-2] - isDir = true - nextParentDir = fileName - if fileName == parentDir || (fileName+"/") == setting.OutPutPath { + prefixLen := len(prefix) + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + var isDir bool + if val.Key == prefix { continue } + if strings.HasSuffix(val.Key, "/") { + isDir = true + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: val.Key[prefixLen:], + Size: val.Size, + IsDir: isDir, + ParenDir: "", + } + fileInfos = append(fileInfos, fileInfo) + } + if output.IsTruncated { + input.Marker = output.NextMarker } else { - fileName = str1[len(str1)-1] - isDir = false + break } - log.Info("fileName=" + fileName) - fileInfo := FileInfo{ - ModTime: val.LastModified.Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: nextParentDir, + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) } - fileInfos = append(fileInfos, fileInfo) + return nil, err } - return fileInfos, err - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) - } - return nil, err } + return fileInfos, nil + } func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index da1bc30ac..807fa192c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -1,6 +1,7 @@ package repo import ( + "archive/zip" "errors" "fmt" "net/http" @@ -105,7 +106,7 @@ func SaveModel(ctx *context.Context) { func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") - modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey, "") + modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") if err != nil { log.Info("get TrainJobListModel failed:", err) return "", 0, err @@ -119,21 +120,6 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix) - // for _, modelFile := range modelDbResult { - // if modelFile.IsDir { - // log.Info("copy dir, continue. dir=" + modelFile.FileName) - // continue - // } - // srcKeyName := prefix + modelFile.FileName - // log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) - // destKeyName := destKeyNamePrefix + modelFile.FileName - // log.Info("Dest key name=" + destKeyName) - // err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) - // if err != nil { - // log.Info("copy failed.") - // } - // size += modelFile.Size - // } dataActualPath := setting.Bucket + "/" + destKeyNamePrefix return dataActualPath, size, nil } @@ -177,10 +163,6 @@ func DeleteModelByID(id string) error { return err } -func DownloadModel(ctx *context.Context) { - log.Info("download model start.") -} - func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) { return models.QueryModel(&models.AiModelQueryOptions{ @@ -198,6 +180,56 @@ func DownloadMultiModelFile(ctx *context.Context) { log.Info("DownloadMultiModelFile start.") id := ctx.Query("ID") log.Info("id=" + id) + task, err := models.QueryModelById(id) + if err != nil { + log.Error("no such model!", err.Error()) + ctx.ServerError("no such model:", err) + return + } + path := Model_prefix + models.AttachmentRelativePath(id) + "/" + returnFileName := task.Name + "_" + task.Version + ".zip" + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) + if err != nil { + + w := zip.NewWriter(ctx.Resp) + defer w.Close() + + for _, oneFile := range allFile { + if !oneFile.IsDir { + log.Info("zip file name:" + oneFile.FileName) + fDest, err := w.Create(oneFile.FileName) + if err != nil { + log.Info("create zip entry error, download file failed: %s\n", err.Error()) + ctx.ServerError("download file failed:", err) + return + } + body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName) + if err != nil { + log.Info("download file failed: %s\n", err.Error()) + ctx.ServerError("download file failed:", err) + return + } else { + defer body.Close() + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + fDest.Write(p[:readCount]) + } + if readErr != nil { + break + } + } + } + + } + } + } } func DownloadSingleModelFile(ctx *context.Context) { @@ -206,13 +238,39 @@ func DownloadSingleModelFile(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") path := Model_prefix + models.AttachmentRelativePath(id) + "/" + parentDir + fileName - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) - if err != nil { - log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrl", err) - return + + if setting.PROXYURL != "" { + body, err := storage.ObsDownloadAFile(setting.Bucket, path) + if err != nil { + log.Info("download error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } + } else { + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } func ShowSingleModel(ctx *context.Context) { @@ -226,7 +284,7 @@ func ShowSingleModel(ctx *context.Context) { return } log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:]) - models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir) + models, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir) if err != nil { log.Info("get model list failed:", err) ctx.ServerError("GetObsListObject:", err) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index aaa2bd6a8..6887c0bd9 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -966,6 +966,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile) }) + m.Get("/downloadall", reqRepoCloudBrainReader, repo.DownloadMultiModelFile) }, context.RepoRef()) m.Group("/modelarts", func() { diff --git a/templates/repo/modelmanage/download.tmpl b/templates/repo/modelmanage/download.tmpl index 72e30c1ee..e36b0e0b5 100644 --- a/templates/repo/modelmanage/download.tmpl +++ b/templates/repo/modelmanage/download.tmpl @@ -16,7 +16,7 @@
- + 下载所有 {{if .Dirs}} From 515b8cf375e84a627394912fac51df02657c3e66 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 17:06:25 +0800 Subject: [PATCH 061/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 1 + routers/repo/ai_model_manage.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index eea716725..c98e570c5 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -341,6 +341,7 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er index++ for _, val := range output.Contents { var isDir bool + log.Info("val.key=" + val.Key) if val.Key == prefix { continue } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 807fa192c..1a267ddd1 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -192,13 +192,11 @@ func DownloadMultiModelFile(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", "application/octet-stream") allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) if err != nil { - w := zip.NewWriter(ctx.Resp) defer w.Close() - for _, oneFile := range allFile { if !oneFile.IsDir { - log.Info("zip file name:" + oneFile.FileName) + log.Info("zip file name:" + oneFile.FileName) fDest, err := w.Create(oneFile.FileName) if err != nil { log.Info("create zip entry error, download file failed: %s\n", err.Error()) From 82179d500d95c66f6dc764b26cf870573529da57 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 17:08:59 +0800 Subject: [PATCH 062/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index c98e570c5..2868e3ec2 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -345,6 +345,7 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er if val.Key == prefix { continue } + if strings.HasSuffix(val.Key, "/") { isDir = true } else { @@ -357,6 +358,7 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er IsDir: isDir, ParenDir: "", } + log.Info("file name=" + val.Key) fileInfos = append(fileInfos, fileInfo) } if output.IsTruncated { From 26ff403ae8014f8875613264e3e9984c2edee0d2 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 17:09:41 +0800 Subject: [PATCH 063/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 2868e3ec2..c0508e81c 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -342,7 +342,7 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er for _, val := range output.Contents { var isDir bool log.Info("val.key=" + val.Key) - if val.Key == prefix { + if prefixLen == len(val.Key) { continue } From da7616ff2b283bb6015738835e26dc1600aca049 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 17:21:22 +0800 Subject: [PATCH 064/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 1a267ddd1..a91df9b69 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -195,7 +195,9 @@ func DownloadMultiModelFile(ctx *context.Context) { w := zip.NewWriter(ctx.Resp) defer w.Close() for _, oneFile := range allFile { - if !oneFile.IsDir { + if oneFile.IsDir { + log.Info("zip dir name:" + oneFile.FileName) + } else { log.Info("zip file name:" + oneFile.FileName) fDest, err := w.Create(oneFile.FileName) if err != nil { @@ -224,7 +226,6 @@ func DownloadMultiModelFile(ctx *context.Context) { } } } - } } } From f6d57940949334965eefc289d73be777f2ba502f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 17:24:42 +0800 Subject: [PATCH 065/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index a91df9b69..fe0be552c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -187,11 +187,12 @@ func DownloadMultiModelFile(ctx *context.Context) { return } path := Model_prefix + models.AttachmentRelativePath(id) + "/" - returnFileName := task.Name + "_" + task.Version + ".zip" - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) - if err != nil { + if err == nil { + returnFileName := task.Name + "_" + task.Version + ".zip" + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") w := zip.NewWriter(ctx.Resp) defer w.Close() for _, oneFile := range allFile { @@ -228,6 +229,9 @@ func DownloadMultiModelFile(ctx *context.Context) { } } } + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) } } From 9c488c4876a676cf463173886882c05272c2387f Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 10 Nov 2021 14:26:00 +0800 Subject: [PATCH 066/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index c0508e81c..9be4a7455 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -341,11 +341,9 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er index++ for _, val := range output.Contents { var isDir bool - log.Info("val.key=" + val.Key) if prefixLen == len(val.Key) { continue } - if strings.HasSuffix(val.Key, "/") { isDir = true } else { @@ -358,7 +356,6 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er IsDir: isDir, ParenDir: "", } - log.Info("file name=" + val.Key) fileInfos = append(fileInfos, fileInfo) } if output.IsTruncated { @@ -374,9 +371,7 @@ func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, er return nil, err } } - return fileInfos, nil - } func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { From c9bc52a8a6b5fb05332e64476cdf38ab9c665ffd Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 10 Nov 2021 15:04:32 +0800 Subject: [PATCH 067/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index c1009122c..92ab9ba2e 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -236,13 +236,8 @@ func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPa log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { - // if strings.HasSuffix(val.Key, "/") { - // log.Info("copy file, src key=" + val.Key + " is dir, not copy.") - // continue - // } destKey := destPath + val.Key[length:] - log.Info("copy file,bucket=" + srcBucket + " src key=" + val.Key + "destbucket=" + destBucket + " dest key=" + destKey) - ObsCopyFile(srcBucket, val.Key, destBucket, destKey) + obsCopyFile(srcBucket, val.Key, destBucket, destKey) fileTotalSize += val.Size } if output.IsTruncated { @@ -261,7 +256,7 @@ func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPa return fileTotalSize, nil } -func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { +func obsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { input := &obs.CopyObjectInput{} input.Bucket = destBucket input.Key = destKeyName @@ -271,6 +266,7 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey if err == nil { log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) } else { + log.Info("copy failed,,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) if obsError, ok := err.(obs.ObsError); ok { log.Info(obsError.Code) log.Info(obsError.Message) From c961aa3b5ade96438dc2ee43fe9e69338a6f7b6a Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 10 Nov 2021 15:53:46 +0800 Subject: [PATCH 068/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/serv.go | 1 - routers/repo/http.go | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/routers/private/serv.go b/routers/private/serv.go index c2c6b656c..b76199481 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -125,7 +125,6 @@ func ServCommand(ctx *macaron.Context) { return } } - log.Info("git oper to this.") for _, verb := range ctx.QueryStrings("verb") { // clone_cnt if verb == "git-upload-pack" { go repo.IncreaseCloneCnt() diff --git a/routers/repo/http.go b/routers/repo/http.go index cab29ba05..ad2abf567 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -9,7 +9,6 @@ import ( "bytes" "compress/gzip" gocontext "context" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -313,20 +312,6 @@ func HTTP(ctx *context.Context) { } environ = append(environ, models.ProtectedBranchRepoID+fmt.Sprintf("=%d", repo.ID)) - log.Info("git oper to this 2.") - - dataJson, _ := json.Marshal(ctx.Data) - log.Info("dataJson=" + string(dataJson)) - - reqJson, _ := json.Marshal(ctx.Req.Form) - log.Info("reqJson=" + string(reqJson)) - - log.Info("query go-get:" + ctx.Query("go-get")) - - log.Info("query service:" + ctx.Query("service")) - - paramJson, _ := json.Marshal(ctx.Params) - log.Info("paramJson=" + string(paramJson)) if service == "git-upload-pack" { // clone_cnt go repo.IncreaseCloneCnt() From 705862fcc56bd1d2127448c10143c46b9c0628f9 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 10 Nov 2021 16:32:03 +0800 Subject: [PATCH 069/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 16 +++++++++++++- routers/repo/ai_model_manage.go | 49 ++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index dfeaba287..f710cc7a6 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -22,7 +22,7 @@ type AiModelManage struct { Path string `xorm:"varchar(400) NOT NULL"` ConfigJson string `xorm:"text"` DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` Status int `xorm:"NOT NULL DEFAULT 0"` Accuracy string `xorm:"varchar(1000)"` AttachmentId string `xorm:"NULL"` @@ -114,7 +114,21 @@ func ModifyModelNewProperty(id string, new int) error { } log.Info("success to update new property from db.re=" + fmt.Sprint((re))) return nil +} +func ModifyModelDownloadCount(id string) error { + var sess *xorm.Session + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.Exec("UPDATE `ai_model_manage` SET download_count = download_count + 1 WHERE id = ?", id); err != nil { + return err + } + if err := sess.Commit(); err != nil { + return err + } + return nil } func QueryModelByName(name string, repoId int64) []*AiModelManage { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index fe0be552c..f4710e9be 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -2,6 +2,7 @@ package repo import ( "archive/zip" + "encoding/json" "errors" "fmt" "net/http" @@ -56,20 +57,35 @@ func SaveModelByParameters(jobId string, name string, version string, label stri return err } } - + accuracy := make(map[string]string) + accuracy["F1"] = "" + accuracy["Recall"] = "" + accuracy["Accuracy"] = "" + accuracy["Precision"] = "" + accuracyJson, _ := json.Marshal(accuracy) + log.Info("accuracyJson=" + string(accuracyJson)) + aiTaskJson, _ := json.Marshal(aiTask) + + //taskConfigInfo,err := models.GetCloudbrainByJobIDAndVersionName(jobId,aiTask.VersionName) model := &models.AiModelManage{ - ID: id, - Version: version, - Label: label, - Name: name, - Description: description, - New: MODEL_LATEST, - Type: cloudType, - Path: modelPath, - Size: modelSize, - AttachmentId: aiTask.Uuid, - RepoId: aiTask.RepoID, - UserId: userId, + ID: id, + Version: version, + Label: label, + Name: name, + Description: description, + New: MODEL_LATEST, + Type: cloudType, + Path: modelPath, + Size: modelSize, + AttachmentId: aiTask.Uuid, + RepoId: aiTask.RepoID, + UserId: userId, + CodeBranch: "", + CodeCommitID: aiTask.CommitID, + Engine: aiTask.EngineID, + ConfigJson: "", + TrainTaskInfo: string(aiTaskJson), + Accuracy: string(accuracyJson), } models.SaveModelToDb(model) @@ -190,6 +206,9 @@ func DownloadMultiModelFile(ctx *context.Context) { allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) if err == nil { + //count++ + models.ModifyModelDownloadCount(id) + returnFileName := task.Name + "_" + task.Version + ".zip" ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) ctx.Resp.Header().Set("Content-Type", "application/octet-stream") @@ -247,6 +266,8 @@ func DownloadSingleModelFile(ctx *context.Context) { if err != nil { log.Info("download error.") } else { + //count++ + models.ModifyModelDownloadCount(id) defer body.Close() ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) ctx.Resp.Header().Set("Content-Type", "application/octet-stream") @@ -272,6 +293,8 @@ func DownloadSingleModelFile(ctx *context.Context) { ctx.ServerError("GetObsCreateSignedUrl", err) return } + //count++ + models.ModifyModelDownloadCount(id) http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } } From adb6038cd3c267088a3bdabf966246ea2f771862 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 10 Nov 2021 17:22:50 +0800 Subject: [PATCH 070/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 8 +++++--- routers/private/tool.go | 5 +++-- routers/repo/ai_model_manage.go | 16 ++++++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index f710cc7a6..0103c4633 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -14,6 +14,7 @@ type AiModelManage struct { ID string `xorm:"pk"` Name string `xorm:"NOT NULL"` Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` New int `xorm:"NOT NULL"` Type int `xorm:"NOT NULL"` Size int64 `xorm:"NOT NULL"` @@ -102,12 +103,13 @@ func ModifyModelDescription(id string, description string) error { return nil } -func ModifyModelNewProperty(id string, new int) error { +func ModifyModelNewProperty(id string, new int, versioncount int) error { var sess *xorm.Session sess = x.ID(id) defer sess.Close() - re, err := sess.Cols("new").Update(&AiModelManage{ - New: new, + re, err := sess.Cols("new", "version_count").Update(&AiModelManage{ + New: new, + VersionCount: versioncount, }) if err != nil { return err diff --git a/routers/private/tool.go b/routers/private/tool.go index bccbc50f3..ef96c1d98 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -47,14 +47,15 @@ func RepoStatisticManually(ctx *macaron.Context) { } func CreateModel(ctx *macaron.Context) { - trainTaskId := ctx.Query("TrainTask") + JobId := ctx.Query("JobId") + VersionName := ctx.Query("VersionName") name := ctx.Query("Name") version := ctx.Query("Version") label := ctx.Query("Label") description := ctx.Query("Description") userId := ctx.QueryInt64("userId") - repo.SaveModelByParameters(trainTaskId, name, version, label, description, userId) + repo.SaveModelByParameters(JobId, VersionName, name, version, label, description, userId) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index f4710e9be..5dcaa6d02 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -25,8 +25,9 @@ const ( MODEL_NOT_LATEST = 0 ) -func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { - aiTask, err := models.GetCloudbrainByJobID(jobId) +func SaveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64) error { + aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) + //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { log.Info("query task error." + err.Error()) return err @@ -70,6 +71,7 @@ func SaveModelByParameters(jobId string, name string, version string, label stri model := &models.AiModelManage{ ID: id, Version: version, + VersionCount: len(aimodels) + 1, Label: label, Name: name, Description: description, @@ -91,8 +93,8 @@ func SaveModelByParameters(jobId string, name string, version string, label stri models.SaveModelToDb(model) if len(lastNewModelId) > 0 { - //udpate status - models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST) + //udpate status and version count + models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST, 0) } log.Info("save model end.") @@ -103,12 +105,13 @@ func SaveModelByParameters(jobId string, name string, version string, label stri func SaveModel(ctx *context.Context) { log.Info("save model start.") JobId := ctx.Query("JobId") + VersionName := ctx.Query("VersionName") name := ctx.Query("Name") version := ctx.Query("Version") label := ctx.Query("Label") description := ctx.Query("Description") - err := SaveModelByParameters(JobId, name, version, label, description, ctx.User.ID) + err := SaveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) if err != nil { log.Info("save model error." + err.Error()) @@ -171,7 +174,8 @@ func DeleteModelByID(id string) error { if model.New == MODEL_LATEST { aimodels := models.QueryModelByName(model.Name, model.RepoId) if len(aimodels) > 0 { - models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST) + //udpate status and version count + models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST, len(aimodels)) } } } From 257fdc07a63a1b95c45ea4c10daa460b14fcc826 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 12 Nov 2021 14:31:16 +0800 Subject: [PATCH 071/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 1 - models/user_business_analysis.go | 115 ++++++++++++++++++++++++++++++++++++- routers/repo/ai_model_manage.go | 3 +- routers/repo/user_data_analysis.go | 30 ++++++++++ routers/routes/routes.go | 1 + 5 files changed, 146 insertions(+), 4 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 0103c4633..9c29b6c7a 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -21,7 +21,6 @@ type AiModelManage struct { Description string `xorm:"varchar(2000)"` Label string `xorm:"varchar(1000)"` Path string `xorm:"varchar(400) NOT NULL"` - ConfigJson string `xorm:"text"` DownloadCount int `xorm:"NOT NULL DEFAULT 0"` Engine int64 `xorm:"NOT NULL DEFAULT 0"` Status int `xorm:"NOT NULL DEFAULT 0"` diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index bb6726a2c..77b569af2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -5,7 +5,9 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" ) type UserBusinessAnalysis struct { @@ -71,6 +73,14 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` } +type UserBusinessAnalysisQueryOptions struct { + ListOptions + UserName string + SortType string + StartTime int64 + EndTime int64 +} + func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) statictisSess := xStatistic.NewSession() @@ -114,11 +124,114 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis return userBusinessAnalysisReturnList } +func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { + + log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + currentTimeNow := time.Now() + pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + + var cond = builder.NewCond() + if len(opts.UserName) > 0 { + cond = cond.And( + builder.Eq{"name": opts.UserName}, + ) + } + cond = cond.And( + builder.Gte{"count_date": fmt.Sprint(pageStartTime)}, + ) + cond = cond.And( + builder.Lte{"count_date": fmt.Sprint(pageEndTime)}, + ) + + count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + statictisSess.Limit(opts.PageSize, start) + } + statictisSess.OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0, setting.UI.IssuePagingNum) + if err := statictisSess.Table("user_business_analysis").Where(cond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + resultMap := make(map[int64]*UserBusinessAnalysis) + + var newAndCond = builder.NewCond() + var newOrCond = builder.NewCond() + for _, userRecord := range userBusinessAnalysisList { + newOrCond.Or( + builder.Eq{"id": userRecord.ID}, + ) + } + newAndCond = newAndCond.And( + newOrCond, + ) + newAndCond = newAndCond.And( + builder.Gte{"count_date": fmt.Sprint(opts.StartTime)}, + ) + newAndCond = newAndCond.And( + builder.Lte{"count_date": fmt.Sprint(opts.EndTime)}, + ) + + userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) + if err := statictisSess.Table("user_business_analysis").Where(newAndCond). + Find(&userBusinessAnalysisList); err != nil { + return nil, 0 + } + + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) + index := 0 + for _, v := range resultMap { + userBusinessAnalysisReturnList[index] = v + index += 1 + } + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList, count +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user") + sess.Select("`user`.*").Table("user").Where("type=1 and is_active=true") userList := make([]*User, 0) sess.Find(&userList) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 5dcaa6d02..1143c01bc 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -82,10 +82,9 @@ func SaveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: userId, - CodeBranch: "", + CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, Engine: aiTask.EngineID, - ConfigJson: "", TrainTaskInfo: string(aiTaskJson), Accuracy: string(accuracyJson), } diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 68cccd478..e24416d4f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) func QueryUserStaticData(ctx *context.Context) { @@ -19,7 +20,36 @@ func QueryUserStaticData(ctx *context.Context) { endTime, _ := time.Parse("2006-01-02", endDate) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) +} + +func QueryUserStaticDataPage(ctx *context.Context) { + startDate := ctx.Query("startDate") + endDate := ctx.Query("endDate") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + userName := ctx.Query("userName") + log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) + startTime, _ := time.Parse("2006-01-02", startDate) + endTime, _ := time.Parse("2006-01-02", endDate) + log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) + + pageOpts := &models.UserBusinessAnalysisQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + UserName: userName, + StartTime: startTime.Unix(), + EndTime: endTime.Unix(), + } + mapInterface := make(map[string]interface{}) + re, count := models.QueryUserStaticDataPage(pageOpts) + mapInterface["data"] = re + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) } func TimingCountDataByDate(date string) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 16cd26644..3bff311d5 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,6 +792,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) m.Get("/tool/query_user_static", repo.QueryUserStaticData) + m.Get("/tool/query_user_static_page", repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Group("/milestone", func() { From 2b0a3b5c933c24f9f0eccdd13d22ef160271b602 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 12 Nov 2021 14:39:08 +0800 Subject: [PATCH 072/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 04961c2d6..b9848815c 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -389,10 +389,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } fileName = str1[len(str1)-2] isDir = true -<<<<<<< HEAD - nextParentDir = fileName - if fileName == parentDir || (fileName+"/") == setting.OutPutPath { -======= if parentDir == "" { nextParentDir = fileName } else { @@ -400,7 +396,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { ->>>>>>> origin/liuzx_trainjob continue } } else { @@ -417,8 +412,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { FileName: fileName, Size: val.Size, IsDir: isDir, -<<<<<<< HEAD -======= ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) @@ -475,7 +468,6 @@ func GetVersionObsListObject(jobName, parentDir string) ([]FileInfo, error) { FileName: fileName, Size: val.Size, IsDir: isDir, ->>>>>>> origin/liuzx_trainjob ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) @@ -514,14 +506,9 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { input := &obs.CreateSignedUrlInput{} -<<<<<<< HEAD input.Bucket = bucket input.Key = key -======= - input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") ->>>>>>> origin/liuzx_trainjob input.Expires = 60 * 60 input.Method = obs.HttpMethodGet comma := strings.LastIndex(key, "/") From 294077e2fe583eb35a7d1cf03df6c4976e1c66a0 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 12 Nov 2021 15:09:10 +0800 Subject: [PATCH 073/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 9c29b6c7a..a210124a4 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -120,15 +120,11 @@ func ModifyModelNewProperty(id string, new int, versioncount int) error { func ModifyModelDownloadCount(id string) error { var sess *xorm.Session defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } + if _, err := sess.Exec("UPDATE `ai_model_manage` SET download_count = download_count + 1 WHERE id = ?", id); err != nil { return err } - if err := sess.Commit(); err != nil { - return err - } + return nil } From d566153017da758d234dea216669627b566e95c0 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 12 Nov 2021 15:11:58 +0800 Subject: [PATCH 074/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a210124a4..a982e565a 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -118,9 +118,8 @@ func ModifyModelNewProperty(id string, new int, versioncount int) error { } func ModifyModelDownloadCount(id string) error { - var sess *xorm.Session + sess := x.NewSession() defer sess.Close() - if _, err := sess.Exec("UPDATE `ai_model_manage` SET download_count = download_count + 1 WHERE id = ?", id); err != nil { return err } From 8908c9f43c1538f91f226be1369930969f56c7b5 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 09:41:38 +0800 Subject: [PATCH 075/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 58 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 77b569af2..a7b2f299e 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -231,7 +231,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type=1 and is_active=true") + sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") userList := make([]*User, 0) sess.Find(&userList) @@ -267,6 +267,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) + OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -361,6 +362,12 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.LoginCount = LoginCountMap[dateRecord.ID] } + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = int(OpenIIndexMap[dateRecord.ID] * 100) + } + dateRecord.CommitModelCount = 0 statictisSess.Insert(&dateRecord) @@ -545,6 +552,55 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoStatisticList := make([]*RepoStatistic, 0) + statictisSess.Find(&repoStatisticList) + repoOpenIIndexMap := make(map[int64]float64) + log.Info("query repo_statistic size=" + fmt.Sprint(len(repoStatisticList))) + for _, repoRecord := range repoStatisticList { + if _, ok := repoOpenIIndexMap[repoRecord.RepoID]; !ok { + repoOpenIIndexMap[repoRecord.RepoID] = repoRecord.RadarTotal + } + } + + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + + userMap := make(map[int64]float64) + + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := userMap[repoRecord.OwnerID]; !ok { + if _, ok := repoOpenIIndexMap[repoRecord.ID]; !ok { + userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] + } + } + } + + //query collaboration + sess.Select("repo_id,user_id,mode").Table("collaboration") + collaborationList := make([]*Collaboration, 0) + sess.Find(&collaborationList) + for _, collaborationRecord := range collaborationList { + if _, ok := userMap[collaborationRecord.UserID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; !ok { + userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] + } + } else { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; !ok { + userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] + } + } + } + return userMap +} + func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() From 73b9233e5e696b4ff15ae966ba904b7c30393517 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 10:00:31 +0800 Subject: [PATCH 076/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a7b2f299e..6c87f8133 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -568,7 +568,7 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false") repoList := make([]*Repository, 0) sess.Find(&repoList) From 04d577cb1fdbb08b7dde2f602ba79aa10563c5eb Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 10:23:41 +0800 Subject: [PATCH 077/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 8412ab466..79490c8d0 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -426,6 +426,62 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } } +func GetVersionObsListObject(jobName, parentDir string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = setting.Bucket + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + strPrefix := strings.Split(input.Prefix, "/") + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + if err == nil { + for _, val := range output.Contents { + str1 := strings.Split(val.Key, "/") + var isDir bool + var fileName, nextParentDir string + if strings.HasSuffix(val.Key, "/") { + //dirs in next level dir + if len(str1)-len(strPrefix) > 2 { + continue + } + fileName = str1[len(str1)-2] + isDir = true + if parentDir == "" { + nextParentDir = fileName + } else { + nextParentDir = parentDir + "/" + fileName + } + + if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { + continue + } + } else { + //files in next level dir + if len(str1)-len(strPrefix) > 1 { + continue + } + fileName = str1[len(str1)-1] + isDir = false + nextParentDir = parentDir + } + + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: nextParentDir, + } + fileInfos = append(fileInfos, fileInfo) + } + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } +} + func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} From c74a8f76369ec13ddf3ab7c72cce526b3a716751 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 10:40:04 +0800 Subject: [PATCH 078/206] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E7=BB=84=E7=BB=87=E7=94=A8=E6=88=B7=E7=9A=84=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 0b9a0d552..9cb0545b7 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -1,6 +1,8 @@ package models import ( + "fmt" + "code.gitea.io/gitea/modules/log" "xorm.io/xorm" ) @@ -12,6 +14,7 @@ type CustomMigration struct { var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, + {"Delete zuzhi user history data ", deleteNotDisplayUser}, } var customMigrationsStatic = []CustomMigration{} @@ -47,3 +50,19 @@ func syncTopicStruct(x *xorm.Engine) error { _, err := x.Exec(query) return err } + +func deleteNotDisplayUser(x *xorm.Engine) error { + + sess := x.NewSession() + defer sess.Close() + sess.Select("`user`.*").Table("user").Where("type == 1") + userList := make([]*User, 0) + + for i, userRecord := range userList { + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + deleteSql := "delete from user_business_analysis where id=" + fmt.Sprint(userRecord.ID) + " and name='" + userRecord.Name + "'" + xStatistic.Exec(deleteSql) + } + + return nil +} From 6709d0f8326dab7c4a16b35cf82253a9864641ff Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 10:58:29 +0800 Subject: [PATCH 079/206] =?UTF-8?q?=E5=A2=9E=E5=8A=A0openi=E6=8C=87?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 6c87f8133..ee5f2ccfb 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "fmt" "time" @@ -598,6 +599,10 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 } } } + + userMapJson, _ := json.Marshal(userMap) + log.Info("userMapJson=" + string(userMapJson)) + return userMap } From 6a2b296825f08015f35e537de96d5e663ea7c135 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:07:01 +0800 Subject: [PATCH 080/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 5 +++-- models/user_business_analysis.go | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 9cb0545b7..0bd52da67 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -14,10 +14,11 @@ type CustomMigration struct { var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, - {"Delete zuzhi user history data ", deleteNotDisplayUser}, } -var customMigrationsStatic = []CustomMigration{} +var customMigrationsStatic = []CustomMigration{ + {"Delete zuzhi user history data ", deleteNotDisplayUser}, +} func MigrateCustom(x *xorm.Engine) { diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ee5f2ccfb..ac51c2816 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -578,7 +578,7 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 log.Info("query Repository size=" + fmt.Sprint(len(repoList))) for _, repoRecord := range repoList { if _, ok := userMap[repoRecord.OwnerID]; !ok { - if _, ok := repoOpenIIndexMap[repoRecord.ID]; !ok { + if _, ok := repoOpenIIndexMap[repoRecord.ID]; ok { userMap[repoRecord.OwnerID] = repoOpenIIndexMap[repoRecord.ID] } } @@ -588,13 +588,16 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 sess.Select("repo_id,user_id,mode").Table("collaboration") collaborationList := make([]*Collaboration, 0) sess.Find(&collaborationList) + + log.Info("query collaborationList size=" + fmt.Sprint(len(collaborationList))) + for _, collaborationRecord := range collaborationList { if _, ok := userMap[collaborationRecord.UserID]; !ok { - if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { userMap[collaborationRecord.UserID] = repoOpenIIndexMap[collaborationRecord.RepoID] } } else { - if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; !ok { + if _, ok := repoOpenIIndexMap[collaborationRecord.RepoID]; ok { userMap[collaborationRecord.UserID] += repoOpenIIndexMap[collaborationRecord.RepoID] } } From 2472cc1deebfeea885107b7efcfde99403fe95ef Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:13:24 +0800 Subject: [PATCH 081/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 0bd52da67..86901c605 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -56,9 +56,9 @@ func deleteNotDisplayUser(x *xorm.Engine) error { sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type == 1") + sess.Select("`user`.*").Table("user").Where("type=1") userList := make([]*User, 0) - + log.Info("delete zuzi user size=" + fmt.Sprint(len(userList))) for i, userRecord := range userList { log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) deleteSql := "delete from user_business_analysis where id=" + fmt.Sprint(userRecord.ID) + " and name='" + userRecord.Name + "'" From 6aa1875c49017e391e485f819439f28a207607fb Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:20:52 +0800 Subject: [PATCH 082/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 86901c605..43dd57b4b 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -52,7 +52,7 @@ func syncTopicStruct(x *xorm.Engine) error { return err } -func deleteNotDisplayUser(x *xorm.Engine) error { +func deleteNotDisplayUser(static *xorm.Engine) error { sess := x.NewSession() defer sess.Close() @@ -62,7 +62,7 @@ func deleteNotDisplayUser(x *xorm.Engine) error { for i, userRecord := range userList { log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) deleteSql := "delete from user_business_analysis where id=" + fmt.Sprint(userRecord.ID) + " and name='" + userRecord.Name + "'" - xStatistic.Exec(deleteSql) + static.Exec(deleteSql) } return nil From d2fb777b2df1500a30070c1ce6dd77b612f6f382 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:24:17 +0800 Subject: [PATCH 083/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 43dd57b4b..d5a8299d9 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -10,6 +10,7 @@ import ( type CustomMigration struct { Description string Migrate func(*xorm.Engine) error + MigrateAll func(*xorm.Engine, *xorm.Engine) error } var customMigrations = []CustomMigration{ @@ -33,10 +34,10 @@ func MigrateCustom(x *xorm.Engine) { } -func MigrateCustomStatic(x *xorm.Engine) { +func MigrateCustomStatic(x *xorm.Engine, static *xorm.Engine) { for _, m := range customMigrationsStatic { log.Info("Migration: %s", m.Description) - if err := m.Migrate(x); err != nil { + if err := m.MigrateAll(x, static); err != nil { log.Error("Migration: %v", err) @@ -52,7 +53,7 @@ func syncTopicStruct(x *xorm.Engine) error { return err } -func deleteNotDisplayUser(static *xorm.Engine) error { +func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { sess := x.NewSession() defer sess.Close() From 8bf634386bb05b70621e3a86ef309bbfe5a04302 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:26:38 +0800 Subject: [PATCH 084/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index d5a8299d9..4d100aa92 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -10,14 +10,18 @@ import ( type CustomMigration struct { Description string Migrate func(*xorm.Engine) error - MigrateAll func(*xorm.Engine, *xorm.Engine) error +} + +type CustomMigrationStatic struct { + Description string + Migrate func(*xorm.Engine, *xorm.Engine) error } var customMigrations = []CustomMigration{ {"Custom v1 Topic struct change to support chinese", syncTopicStruct}, } -var customMigrationsStatic = []CustomMigration{ +var customMigrationsStatic = []CustomMigrationStatic{ {"Delete zuzhi user history data ", deleteNotDisplayUser}, } @@ -37,7 +41,7 @@ func MigrateCustom(x *xorm.Engine) { func MigrateCustomStatic(x *xorm.Engine, static *xorm.Engine) { for _, m := range customMigrationsStatic { log.Info("Migration: %s", m.Description) - if err := m.MigrateAll(x, static); err != nil { + if err := m.Migrate(x, static); err != nil { log.Error("Migration: %v", err) From fe622b26e30d918980eba7db05d43a60ea71ff3a Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:27:54 +0800 Subject: [PATCH 085/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/models.go b/models/models.go index 020ee6d6e..6631a9124 100755 --- a/models/models.go +++ b/models/models.go @@ -232,7 +232,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { return fmt.Errorf("newEngine statistic failed: %v", err) } - MigrateCustomStatic(xStatistic) + MigrateCustomStatic(x, xStatistic) HasEngine = true From 8741ef875b78401769afb80e338f8285575ca983 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:43:23 +0800 Subject: [PATCH 086/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 4d100aa92..8476e405a 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -59,14 +59,17 @@ func syncTopicStruct(x *xorm.Engine) error { func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { - sess := x.NewSession() - defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type=1") - userList := make([]*User, 0) - log.Info("delete zuzi user size=" + fmt.Sprint(len(userList))) - for i, userRecord := range userList { - log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - deleteSql := "delete from user_business_analysis where id=" + fmt.Sprint(userRecord.ID) + " and name='" + userRecord.Name + "'" + querySQL := "select id,name from user where type=1" + + rows, err := x.Query(querySQL) + if err != nil { + log.Info("select db failed,err:", err) + return err + } + + for i, userRow := range rows { + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["id"])) + deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" static.Exec(deleteSql) } From fd928bf5a57b1f1f30989e7256873a976ee001fd Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 15 Nov 2021 11:46:47 +0800 Subject: [PATCH 087/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/custom_migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/custom_migrations.go b/models/custom_migrations.go index 8476e405a..dd72e7934 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -59,7 +59,7 @@ func syncTopicStruct(x *xorm.Engine) error { func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { - querySQL := "select id,name from user where type=1" + querySQL := "select id,name from public.user where type=1" rows, err := x.Query(querySQL) if err != nil { From 0d7d6b59a4155b9dfac3482c13cc9063fd4d4086 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 16 Nov 2021 11:42:00 +0800 Subject: [PATCH 088/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- go.mod | 1 + go.sum | 5 +++ models/custom_migrations.go | 14 +++++-- models/user_business_analysis.go | 58 ++++++++++++++++++------- routers/repo/user_data_analysis.go | 86 ++++++++++++++++++++++++++++++++++---- routers/routes/routes.go | 4 +- 6 files changed, 139 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 9f93281c3..697e61728 100755 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( gitea.com/macaron/macaron v1.4.0 gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 + github.com/360EntSecGroup-Skylar/excelize v1.4.1 // indirect github.com/BurntSushi/toml v0.3.1 github.com/PuerkitoBio/goquery v1.5.0 github.com/RichardKnop/machinery v1.6.9 diff --git a/go.sum b/go.sum index 20096f4bf..a85c68479 100755 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7 h1:N9QFoeNsUXLhl14m gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7/go.mod h1:kgsbFPPS4P+acDYDOPDa3N4IWWOuDJt5/INKRUz7aks= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= +github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0gHwBYyfo5Jcjv/Ks= +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -538,6 +540,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c h1:3wkDRdxK92dF+c1ke2dtj7ZzemFWBHB9plnJOtlwdFA= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= @@ -713,6 +717,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/models/custom_migrations.go b/models/custom_migrations.go index dd72e7934..1b3802ef5 100644 --- a/models/custom_migrations.go +++ b/models/custom_migrations.go @@ -22,6 +22,7 @@ var customMigrations = []CustomMigration{ } var customMigrationsStatic = []CustomMigrationStatic{ + {"Alter user static table field type open_i_index ", alterUserStaticTable}, {"Delete zuzhi user history data ", deleteNotDisplayUser}, } @@ -57,10 +58,17 @@ func syncTopicStruct(x *xorm.Engine) error { return err } +func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error { + alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision" + + _, err := static.Exec(alterSql) + return err + +} + func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { querySQL := "select id,name from public.user where type=1" - rows, err := x.Query(querySQL) if err != nil { log.Info("select db failed,err:", err) @@ -68,8 +76,8 @@ func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error { } for i, userRow := range rows { - log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["id"])) - deleteSql := "delete from user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" + log.Info("delete zuzi user, i=" + fmt.Sprint(i) + " userName=" + string(userRow["name"])) + deleteSql := "delete from public.user_business_analysis where id=" + string(userRow["id"]) + " and name='" + string(userRow["name"]) + "'" static.Exec(deleteSql) } diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index ac51c2816..25e6dcc2f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -3,6 +3,7 @@ package models import ( "encoding/json" "fmt" + "sort" "time" "code.gitea.io/gitea/modules/log" @@ -65,13 +66,15 @@ type UserBusinessAnalysis struct { LoginCount int `xorm:"NOT NULL DEFAULT 0"` //openi index - OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` //user Email string `xorm:"NOT NULL"` //user Name string `xorm:"NOT NULL"` + + DataDate string `xorm:"NOT NULL"` } type UserBusinessAnalysisQueryOptions struct { @@ -82,6 +85,22 @@ type UserBusinessAnalysisQueryOptions struct { EndTime int64 } +type UserBusinessAnalysisList []*UserBusinessAnalysis + +func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisList) Less(i, j int) bool { + if ulist[i].CommitCount > ulist[j].CommitCount { + return true + } else { + if ulist[i].CommitCount == ulist[j].CommitCount { + return ulist[i].ID > ulist[j].ID + } else { + return false + } + } +} + func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) statictisSess := xStatistic.NewSession() @@ -115,12 +134,11 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis } } - userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) - index := 0 + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} for _, v := range resultMap { - userBusinessAnalysisReturnList[index] = v - index += 1 + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) } + sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList } @@ -138,7 +156,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus var cond = builder.NewCond() if len(opts.UserName) > 0 { cond = cond.And( - builder.Eq{"name": opts.UserName}, + builder.Like{"name", opts.UserName}, ) } cond = cond.And( @@ -218,17 +236,17 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } } - userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) - index := 0 + userBusinessAnalysisReturnList := UserBusinessAnalysisList{} for _, v := range resultMap { - userBusinessAnalysisReturnList[index] = v - index += 1 + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) } + sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, count } -func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { +func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) { + log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -246,9 +264,12 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() - CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) + if isReCount { + CountDate = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 1, 0, 0, startTime.Location()) + } + DataDate := startTime.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 10) @@ -285,6 +306,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { dateRecord.CodeMergeCount = 0 } else { @@ -366,7 +388,7 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { dateRecord.OpenIIndex = 0 } else { - dateRecord.OpenIIndex = int(OpenIIndexMap[dateRecord.ID] * 100) + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } dateRecord.CommitModelCount = 0 @@ -376,6 +398,10 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { + CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) +} + func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() @@ -507,10 +533,10 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { resultMap := make(map[int64]int) log.Info("query Follow size=" + fmt.Sprint(len(followList))) for _, followRecord := range followList { - if _, ok := resultMap[followRecord.UserID]; !ok { - resultMap[followRecord.UserID] = 1 + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 } else { - resultMap[followRecord.UserID] += 1 + resultMap[followRecord.FollowID] += 1 } } return resultMap diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index e24416d4f..3f529272d 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/360EntSecGroup-Skylar/excelize" ) func QueryUserStaticData(ctx *context.Context) { @@ -18,6 +19,7 @@ func QueryUserStaticData(ctx *context.Context) { log.Info("startDate=" + startDate + " endDate=" + endDate) startTime, _ := time.Parse("2006-01-02", startDate) endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix())) } @@ -29,17 +31,24 @@ func QueryUserStaticDataPage(ctx *context.Context) { if page <= 0 { page = 1 } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } userName := ctx.Query("userName") + IsReturnFile := ctx.QueryBool("IsReturnFile") + log.Info("startDate=" + startDate + " endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page)) startTime, _ := time.Parse("2006-01-02", startDate) endTime, _ := time.Parse("2006-01-02", endDate) + endTime = endTime.AddDate(0, 0, 1) log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) pageOpts := &models.UserBusinessAnalysisQueryOptions{ ListOptions: models.ListOptions{ Page: page, - PageSize: setting.UI.IssuePagingNum, + PageSize: pageSize, }, UserName: userName, StartTime: startTime.Unix(), @@ -49,11 +58,71 @@ func QueryUserStaticDataPage(ctx *context.Context) { re, count := models.QueryUserStaticDataPage(pageOpts) mapInterface["data"] = re mapInterface["count"] = count - ctx.JSON(http.StatusOK, mapInterface) -} -func TimingCountDataByDate(date string) { + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := "用户分析" + index := xlsx.NewSheet(sheetName) + dataHeader := map[string]string{ + //学科 + "A1": "ID", + "B1": "用户名", + "C1": "PR数", + "D1": "提出任务数", + "E1": "评论数", + "F1": "关注项目数", + "G1": "点赞项目数", + "H1": "登录次数", + "I1": "关注者数", + "J1": "commit代码行数", + "K1": "已解决任务数", + "L1": "百科页面贡献次数", + "M1": "创建项目", + "N1": "OpenI指数", + "O1": "用户注册时间", + "P1": "系统统计时间", + } + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + + for i, userRecord := range re { + rows := fmt.Sprint(i + 2) + + xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) + xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) + xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.IssueCount) + xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommentCount) + xlsx.SetCellValue(sheetName, "F"+rows, userRecord.FocusRepoCount) + xlsx.SetCellValue(sheetName, "G"+rows, userRecord.StarRepoCount) + xlsx.SetCellValue(sheetName, "H"+rows, userRecord.LoginCount) + xlsx.SetCellValue(sheetName, "I"+rows, userRecord.WatchedCount) + xlsx.SetCellValue(sheetName, "J"+rows, userRecord.CommitCodeSize) + xlsx.SetCellValue(sheetName, "K"+rows, userRecord.SolveIssueCount) + xlsx.SetCellValue(sheetName, "L"+rows, userRecord.EncyclopediasCount) + xlsx.SetCellValue(sheetName, "M"+rows, userRecord.CreateRepoCount) + xlsx.SetCellValue(sheetName, "N"+rows, userRecord.OpenIIndex) + xlsx.SetCellValue(sheetName, "O"+rows, userRecord.RegistDate.Format("2006-01-02")) + xlsx.SetCellValue(sheetName, "P"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs("./成绩表.xlsx") + if err != nil { + log.Error(err) + } + + } else { + ctx.JSON(http.StatusOK, mapInterface) + } +} + +func TimingCountDataByDateAndReCount(date string, isReCount bool) { t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -100,16 +169,17 @@ func TimingCountDataByDate(date string) { } } //other user info data - models.CounDataByDate(wikiMap, startTime, endTime) + models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount) +} +func TimingCountDataByDate(date string) { + TimingCountDataByDateAndReCount(date, true) } func TimingCountData() { - log.Info("start to time count data") currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") - - TimingCountDataByDate(startTime) + TimingCountDataByDateAndReCount(startTime, false) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index f827685a7..994cbf36c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -792,8 +792,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static", repo.QueryUserStaticData) - m.Get("/tool/query_user_static_page", repo.QueryUserStaticDataPage) + m.Get("/tool/query_user_static", adminReq, repo.QueryUserStaticData) + m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors) From 71c2f52ca8aaf6c4022fd81b7fe8b6ddcdbe5f40 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 16 Nov 2021 14:18:12 +0800 Subject: [PATCH 089/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 3f529272d..b7a408a90 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "github.com/360EntSecGroup-Skylar/excelize" + "github.com/xuri/excelize/v2" ) func QueryUserStaticData(ctx *context.Context) { From 77d6aa9585857fdde442720f757bf0adab54b8f6 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 22 Nov 2021 10:22:04 +0800 Subject: [PATCH 090/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82=E4=BF=AE=E6=94=B9URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/internal.go | 4 ---- routers/repo/ai_model_manage.go | 2 +- routers/repo/user_data_analysis.go | 4 ++-- routers/routes/routes.go | 5 ++++- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/routers/private/internal.go b/routers/private/internal.go index a3ea1c6cd..b029eedf9 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -44,9 +44,5 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues) m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt) m.Post("/tool/repo_stat", RepoStatisticManually) - m.Post("/tool/create_model", CreateModel) - m.Delete("/tool/delete_model", DeleteModel) - m.Get("/tool/show_model", ShowModel) - m.Put("/tool/modify_model", ModifyModel) }, CheckInternalToken) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 1143c01bc..7268ce18a 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -25,7 +25,7 @@ const ( MODEL_NOT_LATEST = 0 ) -func SaveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 5c701eadb..530d4dc94 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -49,13 +49,13 @@ func QueryUserStaticDataPage(ctx *context.Context) { startTime = time.Now() endTime = time.Now() } else { - startTime, _ = time.Parse("2006-01-02", startDate) + startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local) settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime) if startTime.Unix() < settingStartTime.Unix() { startTime = settingStartTime startDate = settingStartTime.Format("2006-01-02") } - endTime, _ = time.Parse("2006-01-02", endDate) + endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local) endTime = endTime.AddDate(0, 0, 1) isAll = false log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + " endDate=" + fmt.Sprint(endTime.Unix())) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 994cbf36c..d4384bb60 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -970,7 +970,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) m.Group("/modelmanage", func() { - m.Get("", reqRepoCloudBrainReader, repo.ShowModelPageInfo) + m.Post("/create_model", repo.SaveModel) + m.Delete("/delete_model", repo.DeleteModel) + m.Put("/modify_model", repo.ModifyModelInfo) + m.Get("/show_model", reqRepoCloudBrainReader, repo.ShowModelPageInfo) m.Group("/:ID", func() { m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel) m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile) From ee4cad003406fcfb7026ac31aa572b375c65b0c3 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 22 Nov 2021 10:23:30 +0800 Subject: [PATCH 091/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82=E4=BF=AE=E6=94=B9URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 7268ce18a..556da9010 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -110,7 +110,7 @@ func SaveModel(ctx *context.Context) { label := ctx.Query("Label") description := ctx.Query("Description") - err := SaveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) if err != nil { log.Info("save model error." + err.Error()) From abaf4db661767f321f19ae2aa22a964c7e32bbb3 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 22 Nov 2021 10:24:45 +0800 Subject: [PATCH 092/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82=E4=BF=AE=E6=94=B9URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/private/tool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routers/private/tool.go b/routers/private/tool.go index ef96c1d98..d05cd4c44 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -5,7 +5,6 @@ package private import ( - "fmt" "net/http" "code.gitea.io/gitea/models" @@ -46,6 +45,7 @@ func RepoStatisticManually(ctx *macaron.Context) { repo.TimingCountDataByDate(date) } +/* func CreateModel(ctx *macaron.Context) { JobId := ctx.Query("JobId") VersionName := ctx.Query("VersionName") @@ -88,3 +88,4 @@ func ModifyModel(ctx *macaron.Context) { ctx.JSON(500, "Failed.") } } +*/ From d85498be5e70213d034837a9f19800bc3efe083f Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 22 Nov 2021 10:57:05 +0800 Subject: [PATCH 093/206] solve conflict --- options/locale/locale_zh-CN.ini | 3 + templates/repo/header.tmpl | 5 + templates/repo/modelarts/notebook/index.tmpl | 181 -------- templates/repo/modelarts/notebook/new.tmpl | 85 +--- templates/repo/modelarts/trainjob/index.tmpl | 178 ------- templates/repo/modelarts/trainjob/new.tmpl | 80 ---- templates/repo/modelarts/trainjob/version_new.tmpl | 81 ---- templates/repo/modelmanage/index.tmpl | 517 ++++++++++----------- web_src/less/openi.less | 180 +++++++ 9 files changed, 446 insertions(+), 864 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 03d9a03e7..4f8043e6f 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -781,6 +781,9 @@ datasets=数据集 datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 +model_manager = 模型管理 + + debug=调试 stop=停止 delete=删除 diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index d5b911a79..85800311d 100755 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -137,6 +137,11 @@ {{svg "octicon-inbox" 16}} {{.i18n.Tr "datasets"}} {{end}} + {{if .Permission.CanRead $.UnitTypeDatasets}} + + {{svg "octicon-inbox" 16}} {{.i18n.Tr "repo.model_manager"}} + + {{end}} {{if .Permission.CanRead $.UnitTypeCloudBrain}} {{svg "octicon-server" 16}} {{.i18n.Tr "repo.cloudbrain"}} diff --git a/templates/repo/modelarts/notebook/index.tmpl b/templates/repo/modelarts/notebook/index.tmpl index 021394a07..512aff2a5 100755 --- a/templates/repo/modelarts/notebook/index.tmpl +++ b/templates/repo/modelarts/notebook/index.tmpl @@ -1,187 +1,6 @@ {{template "base/head" .}} - -
diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 284d6d6c7..3a16cc102 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -1,89 +1,8 @@ {{template "base/head" .}}
diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 6c989adef..51b2b95e3 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -2,184 +2,6 @@ {{template "base/head" .}} {{template "base/head" .}} - -
@@ -201,72 +20,49 @@
{{template "base/alert" .}} -
-
-
- 调试任务 - 训练任务 - 模型管理 -
-
-
- -
- {{svg "octicon-server" 16}} -
Ascend NPU
- -
- CPU / GPU - Ascend NPU -
-
- - 新建调试任务 -
- +
+
+ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.modelarts.train_job.new_train"}}{{end}} +
- - - -
- - +
-
+
{{$.i18n.Tr "repo.cloudbrain_task"}}
-
- {{$.i18n.Tr "repo.cloudbrain_status_createtime"}} +
+ {{$.i18n.Tr "repo.modelarts.version_nums"}} +
+
+ {{$.i18n.Tr "repo.modelarts.version_nums"}} +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}}
-
- {{$.i18n.Tr "repo.cloudbrain_status_runtime"}} +
+ {{$.i18n.Tr "repo.modelarts.createtime"}}
-
+
+ {{$.i18n.Tr "repo.modelarts.computing_resources"}} +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}} +
+
{{$.i18n.Tr "repo.cloudbrain_creator"}}
-
- {{$.i18n.Tr "repo.cloudbrain_operate"}} +
+ {{$.i18n.Tr "repo.cloudbrain_operate"}}
@@ -280,57 +76,86 @@
-
- +
+ {{svg "octicon-tasklist" 16}} - {{.Name}} + {{.JobName}}
- -
- - - - {{.Status}} + +
+ {{.VersionCount}} +
+ +
+ + {{.Status}} - - {{TimeSinceUnix .CreatedUnix $.Lang}}
- -
- - - + +
+ {{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}} +
+ + +
+ +
+ +
+ {{.ComputeResource}} +
+ +
+ {{if .User.Name}} + + {{else}} + + {{end}}
- -
- -
- - - {{$.i18n.Tr "repo.model_download"}} +
+
+ + {{$.CsrfTokenHtml}} + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.stop"}} + + {{else}} + + {{$.i18n.Tr "repo.stop"}} - + {{end}} +
+ + -
+ {{$.CsrfTokenHtml}} {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} - + {{$.i18n.Tr "repo.delete"}} {{else}} - + {{$.i18n.Tr "repo.delete"}} {{end}}
- - - -
{{end}} {{template "base/paginate" .}} @@ -371,3 +196,173 @@
{{template "base/footer" .}} + + diff --git a/web_src/less/openi.less b/web_src/less/openi.less index e073dca0a..892d0adf4 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -316,4 +316,184 @@ display: block; margin-left: 4px !important; color: #3291F8; +} + + +.selectcloudbrain .active.item{ + color: #0087f5 !important; + border: 1px solid #0087f5; + margin: -1px; + background: #FFF !important; +} +#deletemodel { + width: 100%; + height: 100%; +} +/* 弹窗 */ + +#mask { + position: fixed; + top: 0px; + left: 0px; + right: 0px; + bottom: 0px; + filter: alpha(opacity=60); + background-color: #777; + z-index: 1000; + display: none; + opacity: 0.8; + -moz-opacity: 0.5; + padding-top: 100px; + color: #000000 +} + +#loadingPage { + margin: 200px auto; + width: 50px; + height: 40px; + text-align: center; + font-size: 10px; + display: block; +} + +#loadingPage>div { + background-color: green; + height: 100%; + width: 6px; + display: inline-block; + -webkit-animation: sk-stretchdelay 1.2s infinite ease-in-out; + animation: sk-stretchdelay 1.2s infinite ease-in-out; +} + +#loadingPage .rect2 { + -webkit-animation-delay: -1.1s; + animation-delay: -1.1s; +} + +#loadingPage .rect3 { + -webkit-animation-delay: -1.0s; + animation-delay: -1.0s; +} + +#loadingPage .rect4 { + -webkit-animation-delay: -0.9s; + animation-delay: -0.9s; +} + +#loadingPage .rect5 { + -webkit-animation-delay: -0.8s; + animation-delay: -0.8s; +} + +@-webkit-keyframes sk-stretchdelay { + 0%, + 40%, + 100% { + -webkit-transform: scaleY(0.4) + } + 20% { + -webkit-transform: scaleY(1.0) + } +} + +@keyframes sk-stretchdelay { + 0%, + 40%, + 100% { + transform: scaleY(0.4); + -webkit-transform: scaleY(0.4); + } + 20% { + transform: scaleY(1.0); + -webkit-transform: scaleY(1.0); + } +} +/* 消息框 */ + +.alert { + display: none; + position: fixed; + width: 100%; + z-index: 1001; + padding: 15px; + border: 1px solid transparent; + border-radius: 4px; + text-align: center; + font-weight: bold; +} + +.alert-success { + color: #3c763d; + background-color: #dff0d8; + border-color: #d6e9c6; +} + +.alert-info { + color: #31708f; + background-color: #d9edf7; + border-color: #bce8f1; +} + +.alert-warning { + color: #8a6d3b; + background-color: #fcf8e3; + border-color: #faebcc; +} + +.alert-danger { + color: #a94442; + background-color: #f2dede; + border-color: #ebccd1; +} + +.pusher { + width: calc(100% - 260px); + box-sizing: border-box; +} +/* 弹窗 (background) */ + +#imageModal { + display: none; + position: fixed; + z-index: 1; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: auto; + background-color: rgb(0, 0, 0); + background-color: rgba(0, 0, 0, 0.4); +} +/* 弹窗内容 */ + +.modal-content { + background-color: #fefefe; + margin: 15% auto; + padding: 20px; + border: 1px solid #888; + width: 30%; +} +/* 关闭按钮 */ + +.close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; +} + +.close:hover, +.close:focus { + color: black; + text-decoration: none; + cursor: pointer; +} + +.dis { + margin-bottom: 20px; +} + +.disabled { + cursor: pointer; + pointer-events: none; } \ No newline at end of file From d1165f016ab2bc93915c6fb544f5bdc13083a287 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 08:47:12 +0800 Subject: [PATCH 094/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 9e9bf7f3a..e0c864cd2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -209,7 +209,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap := make(map[int64]*UserBusinessAnalysis) - if opts.Page >= 0 && opts.PageSize > 0 && len(userBusinessAnalysisList) > 0 { + if len(userBusinessAnalysisList) > 0 { var newAndCond = builder.NewCond() var newOrCond = builder.NewCond() for _, userRecord := range userBusinessAnalysisList { From 190e275c8c75b955d866718c2c832d181d2386da Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 08:52:02 +0800 Subject: [PATCH 095/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index e0c864cd2..01875a063 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -199,7 +199,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } statictisSess.Limit(opts.PageSize, start) } - statictisSess.OrderBy("count_date desc") + statictisSess.OrderBy("count_date desc, commit_count desc") userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) if err := statictisSess.Table("user_business_analysis").Where(cond). From a8205877ed14c97ce1dfa463a22486870dc64acf Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 08:57:14 +0800 Subject: [PATCH 096/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 01875a063..99a05c031 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -199,10 +199,9 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } statictisSess.Limit(opts.PageSize, start) } - statictisSess.OrderBy("count_date desc, commit_count desc") userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond). + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date desc"). Find(&userBusinessAnalysisList); err != nil { return nil, 0 } @@ -228,9 +227,8 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus builder.Lte{"count_date": opts.EndTime}, ) } - userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond). + if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc"). Find(&userBusinessAnalysisList); err != nil { return nil, 0 } From ad324cea5035a277d20189ee62f441777b29a496 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 11:15:10 +0800 Subject: [PATCH 097/206] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=85=A8=E5=B1=80?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 99a05c031..502cec806 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -90,15 +90,7 @@ type UserBusinessAnalysisList []*UserBusinessAnalysis func (ulist UserBusinessAnalysisList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } func (ulist UserBusinessAnalysisList) Len() int { return len(ulist) } func (ulist UserBusinessAnalysisList) Less(i, j int) bool { - if ulist[i].CommitCount > ulist[j].CommitCount { - return true - } else { - if ulist[i].CommitCount == ulist[j].CommitCount { - return ulist[i].ID > ulist[j].ID - } else { - return false - } - } + return ulist[i].ID > ulist[j].ID } func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { @@ -201,7 +193,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus } userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date desc"). + if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). Find(&userBusinessAnalysisList); err != nil { return nil, 0 } From e216e5bf38c2fc3b0b558d359da48cd18c14dc28 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 11:44:43 +0800 Subject: [PATCH 098/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 530d4dc94..822fc8c30 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -129,8 +129,8 @@ func QueryUserStaticDataPage(ctx *context.Context) { xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) - xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02")) - xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02")) + xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02 10:10")) + xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02 10:10")) } //设置默认打开的表单 From a8a8606e8f3579c84bf2fde1105bc04b195e9cb2 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 23 Nov 2021 11:52:03 +0800 Subject: [PATCH 099/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/user_data_analysis.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 822fc8c30..26ca3ae8f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -129,8 +129,12 @@ func QueryUserStaticDataPage(ctx *context.Context) { xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) - xlsx.SetCellValue(sheetName, "P"+rows, userRecord.RegistDate.Format("2006-01-02 10:10")) - xlsx.SetCellValue(sheetName, "Q"+rows, time.Unix(userRecord.CountDate, 0).Format("2006-01-02 10:10")) + + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) + + formatTime = time.Unix(userRecord.CountDate, 0).Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime[0:len(formatTime)-3]) } //设置默认打开的表单 From f49d6f2d695e4d2fd3e6210f2cf95cc4eeb2d0b5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 23 Nov 2021 15:11:10 +0800 Subject: [PATCH 100/206] update --- templates/repo/modelarts/trainjob/show.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index cb92e3f30..6d8996daa 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -434,7 +434,7 @@ td, th {
- {{end}} + {{end}}{{template "base/paginate" .}}
@@ -460,7 +460,8 @@ td, th { {{template "base/footer" .}} - diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue new file mode 100644 index 000000000..ae71253d6 --- /dev/null +++ b/web_src/js/components/Model.vue @@ -0,0 +1,305 @@ + + + + + diff --git a/web_src/js/index.js b/web_src/js/index.js index 1b718ed53..0f60504de 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -39,6 +39,7 @@ import Images from './components/Images.vue'; import EditTopics from './components/EditTopics.vue'; import DataAnalysis from './components/DataAnalysis.vue' import Contributors from './components/Contributors.vue' +import Model from './components/Model.vue'; Vue.use(ElementUI); Vue.prototype.$axios = axios; @@ -2916,11 +2917,12 @@ $(document).ready(async () => { initVueEditTopic(); initVueContributors(); initVueImages(); + initVueModel(); initVueDataAnalysis(); initTeamSettings(); initCtrlEnterSubmit(); initNavbarContentToggle(); - // initTopicbar(); + // initTopicbar();vim // closeTopicbar(); initU2FAuth(); initU2FRegister(); @@ -3646,7 +3648,7 @@ function initVueContributors() { function initVueImages() { const el = document.getElementById('images'); - console.log("el",el) + if (!el) { return; @@ -3658,6 +3660,20 @@ function initVueImages() { render: h => h(Images) }); } +function initVueModel() { + const el = document.getElementById('model_list'); + + + if (!el) { + return; + } + + new Vue({ + el: el, + + render: h => h(Model) + }); +} function initVueDataAnalysis() { const el = document.getElementById('data_analysis'); console.log("el",el) From 35e7fd55673d88fc2372b033d0b160fe2bd02634 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 30 Nov 2021 14:52:21 +0800 Subject: [PATCH 131/206] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 3 +-- routers/repo/ai_model_manage.go | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 369333faf..400c77b86 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -287,6 +287,7 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative prefixLen := len(input.Prefix) if err == nil { for _, val := range output.Contents { + log.Info("val key=" + val.Key) var isDir bool var fileName string if val.Key == input.Prefix { @@ -454,8 +455,6 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file return output.SignedUrl, nil } - - func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = bucket diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index f1274d0b7..4f07996fe 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -125,6 +125,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") + log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { log.Info("get TrainJobListModel failed:", err) return "", 0, err From 8fc43e4c98fcd759450669a737b866151319ccab Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 30 Nov 2021 15:02:39 +0800 Subject: [PATCH 132/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 400c77b86..8c6ccfa06 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -282,6 +282,9 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefixRootPath + relativePath + if !strings.HasSuffix(input.Prefix, "/") { + input.Prefix += "/" + } output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) prefixLen := len(input.Prefix) From 4fda025cbd1b143c28b60a7e4ba050f150f18bba Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 30 Nov 2021 15:09:12 +0800 Subject: [PATCH 133/206] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E5=88=A4=E6=96=AD=EF=BC=8C=E5=A6=82=E6=9E=9C=E4=B8=BA=E7=A9=BA?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E8=BF=94=E5=9B=9Eerror?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 4f07996fe..90d537031 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -110,6 +110,16 @@ func SaveModel(ctx *context.Context) { label := ctx.Query("Label") description := ctx.Query("Description") + if JobId == "" || VersionName == "" { + ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) + return + } + + if name == "" || version == "" { + ctx.Error(500, fmt.Sprintf("name or version is null.")) + return + } + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) if err != nil { From b217b20472067c396ea3e49e9b888ec71d8eb1bb Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 30 Nov 2021 17:25:14 +0800 Subject: [PATCH 134/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 1 + routers/repo/ai_model_manage.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a982e565a..711ffd53d 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -52,6 +52,7 @@ func SaveModelToDb(model *AiModelManage) error { re, err := sess.Insert(model) if err != nil { + log.Info("insert error." + err.Error()) return err } log.Info("success to save db.re=" + fmt.Sprint((re))) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 90d537031..a5a1a7d9c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -89,8 +89,10 @@ func saveModelByParameters(jobId string, versionName string, name string, versio Accuracy: string(accuracyJson), } - models.SaveModelToDb(model) - + err = models.SaveModelToDb(model) + if err != nil { + return err + } if len(lastNewModelId) > 0 { //udpate status and version count models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST, 0) From 1e0022c9572ab4ab68c6eec84033570b70697b99 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 30 Nov 2021 17:37:55 +0800 Subject: [PATCH 135/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/routes/routes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 3a5d7e460..b7360ae0c 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -974,6 +974,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Put("/modify_model", repo.ModifyModelInfo) m.Get("/show_model", reqRepoCloudBrainReader, repo.ShowModelTemplate) m.Get("/show_model_api", reqRepoCloudBrainReader, repo.ShowModelPageInfo) + m.Get("/show_model_child_api", reqRepoCloudBrainReader, repo.ShowOneVersionOtherModel) m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList) m.Get("/query_train_job_version", reqRepoCloudBrainReader, repo.QueryTrainJobVersionList) m.Group("/:ID", func() { From 791ef137ce54cf16ad23863309d320d9c0fee88f Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 30 Nov 2021 19:42:38 +0800 Subject: [PATCH 136/206] cb_limit --- models/cloudbrain.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++ routers/repo/cloudbrain.go | 17 ++++++++++++++- routers/repo/modelarts.go | 32 +++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index bb1241247..7b196a321 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -31,6 +31,7 @@ const ( JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" + //notebook ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 ModelArtsCreateFailed ModelArtsJobStatus = "CREATE_FAILED" //创建失败 @@ -46,6 +47,30 @@ const ( ModelArtsDeleted ModelArtsJobStatus = "DELETED" //已删除 ModelArtsResizing ModelArtsJobStatus = "RESIZING" //规格变更中 ModelArtsResizFailed ModelArtsJobStatus = "RESIZE_FAILED" //规格变更失败 + + //trainjob + ModelArtsTrainJobUnknown ModelArtsJobStatus = "UNKNOWN" //作业状态未知 + ModelArtsTrainJobInit ModelArtsJobStatus = "INIT" //作业初始化状态 + ModelArtsTrainJobImageCreating ModelArtsJobStatus = "IMAGE_CREATING" //作业镜像正在创建 + ModelArtsTrainJobImageFailed ModelArtsJobStatus = "IMAGE_FAILED" //作业镜像创建失败 + ModelArtsTrainJobSubmitTrying ModelArtsJobStatus = "SUBMIT_TRYING" //作业正在提交 + ModelArtsTrainJobSubmitFailed ModelArtsJobStatus = "SUBMIT_FAILED" //作业提交失败 + ModelArtsTrainJobDeleteFailed ModelArtsJobStatus = "DELETE_FAILED" //作业删除失败 + ModelArtsTrainJobWaiting ModelArtsJobStatus = "WAITING" //作业正在排队中 + ModelArtsTrainJobRunning ModelArtsJobStatus = "RUNNING" //作业正在运行中 + ModelArtsTrainJobKilling ModelArtsJobStatus = "KILLING" //作业正在取消 + ModelArtsTrainJobCompleted ModelArtsJobStatus = "COMPLETED" //作业已经完成 + ModelArtsTrainJobFailed ModelArtsJobStatus = "FAILED" //作业运行失败 + ModelArtsTrainJobKilled ModelArtsJobStatus = "KILLED" //作业取消成功 + ModelArtsTrainJobCanceled ModelArtsJobStatus = "CANCELED" //作业取消 + ModelArtsTrainJobLost ModelArtsJobStatus = "LOST" //作业丢失 + ModelArtsTrainJobScaling ModelArtsJobStatus = "SCALING" //作业正在扩容 + ModelArtsTrainJobSubmitModelFailed ModelArtsJobStatus = "SUBMIT_MODEL_FAILED" //提交模型失败 + ModelArtsTrainJobDeployServiceFailed ModelArtsJobStatus = "DEPLOY_SERVICE_FAILED" //部署服务失败 + ModelArtsTrainJobCheckInit ModelArtsJobStatus = "CHECK_INIT" //审核作业初始化 + ModelArtsTrainJobCheckRunning ModelArtsJobStatus = "CHECK_RUNNING" //审核作业正在运行中 + ModelArtsTrainJobCheckRunningCompleted ModelArtsJobStatus = "CHECK_RUNNING_COMPLETED" //审核作业已经完成 + ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败 ) type Cloudbrain struct { @@ -1091,3 +1116,31 @@ func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { } return false } + +func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0, 10) + return cloudbrains, x. + NotIn("status", + JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, + ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, + ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). + Limit(100). + Find(&cloudbrains) +} + +func GetCloudbrainCountByUserID(userID int64) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainNotebookCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsRestarting). + And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainTrainJobCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted). + And("job_type = ? and user_id = ? and type = ?", JobTypeTrain, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 28f3a0184..d704ee0d9 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -216,7 +216,22 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - _, err := models.GetCloudbrainByName(jobName) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form) + return + } + } + + _, err = models.GetCloudbrainByName(jobName) if err == nil { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 069a1a0b5..263bba85c 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -116,7 +116,22 @@ func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) description := form.Description flavor := form.Flavor - err := modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) + count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsNotebookNew, &form) + return + } + } + + err = modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return @@ -854,6 +869,21 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.Data["PageIsTrainJob"] = true var jobID = ctx.Params(":jobid") + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form) + return + } + } + latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(jobID, modelarts.IsLatestVersion) if err != nil { ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) From 5cf137c1d78101514f449284e1d757aa912f47a3 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 1 Dec 2021 08:49:28 +0800 Subject: [PATCH 137/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 1 + routers/repo/ai_model_manage.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 711ffd53d..00f844881 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -30,6 +30,7 @@ type AiModelManage struct { CodeBranch string `xorm:"varchar(400) NULL"` CodeCommitID string `xorm:"NULL"` UserId int64 `xorm:"NOT NULL"` + UserName string `xorm:"NULL"` TrainTaskInfo string `xorm:"text NULL"` CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index a5a1a7d9c..a02561e02 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -25,7 +25,7 @@ const ( MODEL_NOT_LATEST = 0 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64, userName string) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { @@ -82,6 +82,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio AttachmentId: aiTask.Uuid, RepoId: aiTask.RepoID, UserId: userId, + UserName: userName, CodeBranch: aiTask.BranchName, CodeCommitID: aiTask.CommitID, Engine: aiTask.EngineID, @@ -122,7 +123,7 @@ func SaveModel(ctx *context.Context) { return } - err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID) + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID, ctx.User.Name) if err != nil { log.Info("save model error." + err.Error()) From 97e6e6804a7205671386bea8c11184b9afbe1702 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 1 Dec 2021 09:53:09 +0800 Subject: [PATCH 138/206] mod --- routers/repo/modelarts.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 263bba85c..48978b2e5 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -699,6 +699,21 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCount EngineName := form.EngineName + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + ErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + ErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobNew, &form) + return + } + } + if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) ErrorNewDataPrepare(ctx, form) @@ -872,13 +887,13 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + VersionErrorDataPrepare(ctx, form) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form) return } From b4e096277dbaff3bb7ff08e40c7ddc7360181cc1 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 1 Dec 2021 15:44:00 +0800 Subject: [PATCH 139/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 47 +++++++++++++++++++++-------------------- routers/repo/ai_model_manage.go | 43 +++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 00f844881..8f4fbd113 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -11,29 +11,30 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string `xorm:"NULL"` - TrainTaskInfo string `xorm:"text NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + ID string `xorm:"pk"` + Name string `xorm:"NOT NULL"` + Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` + New int `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + AttachmentId string `xorm:"NULL"` + RepoId int64 `xorm:"NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + UserName string `xorm:"NULL"` + UserRelAvatarLink string `xorm:"NULL"` + TrainTaskInfo string `xorm:"text NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } type AiModelQueryOptions struct { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index a02561e02..e551df260 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -25,7 +25,7 @@ const ( MODEL_NOT_LATEST = 0 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64, userName string) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, userId int64, userName string, userHeadUrl string) error { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) //aiTask, err := models.GetCloudbrainByJobID(jobId) if err != nil { @@ -69,25 +69,26 @@ func saveModelByParameters(jobId string, versionName string, name string, versio //taskConfigInfo,err := models.GetCloudbrainByJobIDAndVersionName(jobId,aiTask.VersionName) model := &models.AiModelManage{ - ID: id, - Version: version, - VersionCount: len(aimodels) + 1, - Label: label, - Name: name, - Description: description, - New: MODEL_LATEST, - Type: cloudType, - Path: modelPath, - Size: modelSize, - AttachmentId: aiTask.Uuid, - RepoId: aiTask.RepoID, - UserId: userId, - UserName: userName, - CodeBranch: aiTask.BranchName, - CodeCommitID: aiTask.CommitID, - Engine: aiTask.EngineID, - TrainTaskInfo: string(aiTaskJson), - Accuracy: string(accuracyJson), + ID: id, + Version: version, + VersionCount: len(aimodels) + 1, + Label: label, + Name: name, + Description: description, + New: MODEL_LATEST, + Type: cloudType, + Path: modelPath, + Size: modelSize, + AttachmentId: aiTask.Uuid, + RepoId: aiTask.RepoID, + UserId: userId, + UserName: userName, + UserRelAvatarLink: userHeadUrl, + CodeBranch: aiTask.BranchName, + CodeCommitID: aiTask.CommitID, + Engine: aiTask.EngineID, + TrainTaskInfo: string(aiTaskJson), + Accuracy: string(accuracyJson), } err = models.SaveModelToDb(model) @@ -123,7 +124,7 @@ func SaveModel(ctx *context.Context) { return } - err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID, ctx.User.Name) + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx.User.ID, ctx.User.Name, ctx.User.RelAvatarLink()) if err != nil { log.Info("save model error." + err.Error()) From ca57bd075c958333396afa78d65309d403d78947 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 1 Dec 2021 15:49:25 +0800 Subject: [PATCH 140/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 5 +++++ routers/routes/routes.go | 1 + 2 files changed, 6 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index e551df260..1e5c9198c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -21,6 +21,7 @@ const ( Model_prefix = "aimodels/" tplModelManageIndex = "repo/modelmanage/index" tplModelManageDownload = "repo/modelmanage/download" + tplModelInfo = "repo/modelmanage/showinfo" MODEL_LATEST = 1 MODEL_NOT_LATEST = 0 ) @@ -347,6 +348,10 @@ func DownloadSingleModelFile(ctx *context.Context) { } } +func ShowModelInfo(ctx *context.Context) { + ctx.HTML(200, tplModelInfo) +} + func ShowSingleModel(ctx *context.Context) { id := ctx.Params(":ID") parentDir := ctx.Query("parentDir") diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 69c892fdc..f30a52975 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -973,6 +973,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Delete("/delete_model", repo.DeleteModel) m.Put("/modify_model", repo.ModifyModelInfo) m.Get("/show_model", reqRepoCloudBrainReader, repo.ShowModelTemplate) + m.Get("/show_model_info", reqRepoCloudBrainReader, repo.ShowModelInfo) m.Get("/show_model_api", reqRepoCloudBrainReader, repo.ShowModelPageInfo) m.Get("/show_model_child_api", reqRepoCloudBrainReader, repo.ShowOneVersionOtherModel) m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList) From d684c8ef3f997b151f80df4e1e5091e881eeda74 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Wed, 1 Dec 2021 16:26:10 +0800 Subject: [PATCH 141/206] fix issue --- templates/repo/modelmanage/index.tmpl | 434 +++++++++++++++++++++++++++++++ templates/repo/modelmanage/showinfo.tmpl | 0 web_src/js/components/Model.vue | 277 ++++++++++++-------- 3 files changed, 607 insertions(+), 104 deletions(-) create mode 100644 templates/repo/modelmanage/showinfo.tmpl diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index e69de29bb..243c48d26 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -0,0 +1,434 @@ + +{{template "base/head" .}} + + +
+
+
+
+
+
+
+
+
+{{$repository := .Repository.ID}} + +
+ +
+ {{template "repo/header" .}} + +
+ {{template "base/alert" .}} +
+
+
+ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.modelarts.train_job.new_train"}}{{end}} +
+
+ + +
+
+
+ +
+ + + + + +
+ +
+
+
+ +
+ +
+ + +
+ + + + +
+
+
+ 删除任务 +
+ +
+

你确认删除该任务么？此任务一旦删除不可恢复。

+
+
+
+ 取消操作 +
+
+ 确定操作 +
+
+
+
+
+
+
+

导入新模型

+
+
+
+
+

+
+ +
+
+ + + + +
+
+ + + +
+
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ +
+
+
+ +
+
+ + +
+ + +{{template "base/footer" .}} + + + diff --git a/templates/repo/modelmanage/showinfo.tmpl b/templates/repo/modelmanage/showinfo.tmpl new file mode 100644 index 000000000..e69de29bb diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index ae71253d6..95806664e 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -3,102 +3,111 @@
${key}${obj[key]}'+description+'${obj[key]}
- - {{range .Dirs}} - - - - - - {{end}} - -
- - - - {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} - - - - - {{.Size | FileSize}} - - - {{.ModTime}} -
- - {{end}} - -
-
- - - - - - - -{{template "base/footer" .}} diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index f668dccee..3640ff133 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -5,17 +5,14 @@ {{if .LatestCommitUser}} {{if .LatestCommitUser.FullName}} - {{.LatestCommitUser.FullName}} - {{.LatestCommit.Committer.Name}} + {{.LatestCommitUser.FullName}} {{else}} - {{if .LatestCommit.Author}}{{.LatestCommit.Author.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} - {{.LatestCommit.Committer.Name}} + {{if .LatestCommit.Author}}{{.LatestCommit.Author.Name}}{{else}}{{.LatestCommitUser.Name}}{{end}} {{end}} {{else}} {{if .LatestCommit.Author}} - {{.LatestCommit.Author.Name}} - {{.LatestCommit.Committer.Name}} + {{.LatestCommit.Author.Name}} {{end}} {{end}} From 2608ea53a3c9d2a6ff09c5215657b3660a8ac37c Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 7 Dec 2021 17:13:33 +0800 Subject: [PATCH 195/206] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E6=9C=80=E7=BB=88=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/setting/setting.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index ce9738f08..0ab1bbc88 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,9 +433,6 @@ var ( AuthUser string AuthPassword string - //home page - RecommentRepoAddr string - //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string @@ -1228,9 +1225,6 @@ func NewContext() { LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") - sec = Cfg.Section("homepage") - RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") - sec = Cfg.Section("cloudbrain") CBAuthUser = sec.Key("USER").MustString("") CBAuthPassword = sec.Key("PWD").MustString("") From c8899105b99ea4339ac5fa119c7011a7be7b42f6 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 7 Dec 2021 18:01:26 +0800 Subject: [PATCH 196/206] opt --- modules/setting/setting.go | 18 ++++++++++++------ routers/repo/cloudbrain.go | 21 +++++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0ab1bbc88..f7b78235b 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -451,18 +451,21 @@ var ( //benchmark config IsBenchmarkEnabled bool - BenchmarkCode string + BenchmarkOwner string + BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetCode string + Snn4imagenetOwner string + Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreCode string + BrainScoreOwner string + BrainScoreName string BrainScoreServerHost string //blockchain config @@ -1238,18 +1241,21 @@ func NewContext() { sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkOwner = sec.Key("OWNER").MustString("") + BenchmarkName = sec.Key("NAME").MustString("") BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetOwner = sec.Key("OWNER").MustString("") + Snn4imagenetName = sec.Key("NAME").MustString("") Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreOwner = sec.Key("OWNER").MustString("") + BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 65a475e5b..5c2f5b5b7 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "os/exec" "regexp" "sort" "strconv" @@ -263,19 +262,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuType = gpuInfo.Value } } - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType) uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } @@ -700,19 +699,21 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { +func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } - command := "git clone " + gitPath + " " + codePath - cmd := exec.Command("/bin/bash", "-c", command) - _, err = cmd.Output() - + repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { - log.Error("exec.Command(%s) failed:%v", command, err) + log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) + return err + } + + if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } From 7fbddc95e0a845241762c631c83c463fdbf66801 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 8 Dec 2021 09:29:29 +0800 Subject: [PATCH 197/206] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=95=8C=E9=9D=A2?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E4=B9=9F=E5=AF=B9=E7=9C=9F=E5=AE=9E=E7=9A=84?= =?UTF-8?q?git=20clone=E6=AC=A1=E6=95=B0=E5=8A=A01?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 437521d5a..a182e9087 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -532,6 +532,7 @@ func Download(ctx *context.Context) { } ctx.Repo.Repository.IncreaseCloneCnt() + ctx.Repo.Repository.IncreaseGitCloneCnt() ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } From 3c416724560310d752464965e4e1ac004aa434f3 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Dec 2021 10:00:37 +0800 Subject: [PATCH 198/206] #687 fix bug --- models/user.go | 2 +- options/locale/locale_zh-CN.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/user.go b/models/user.go index ec96420dc..b362472e8 100755 --- a/models/user.go +++ b/models/user.go @@ -929,7 +929,7 @@ var ( "template", "user", "vendor", - "dashbord", + "dashboard", "operation", "blockchain", "avatar", diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index e26e07903..15e41b7ea 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -848,7 +848,7 @@ modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 modelarts.train_job.parameter_setting_info=参数信息 modelarts.train_job.fast_parameter_setting=一键式参数配置 -modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 +modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 modelarts.train_job.fast_parameter_setting_config_link=这里 modelarts.train_job.frames=常用框架 modelarts.train_job.algorithm_origin=算法来源 @@ -1965,7 +1965,7 @@ team_unit_desc=允许访问项目单元 team_unit_disabled=(已禁用) form.name_reserved=组织名称 '%s' 是被保留的。 -form.name_pattern_not_allowed=项目名称中不允许使用 "%s"。 +form.name_pattern_not_allowed=组织名称中不允许使用 "%s"。 form.create_org_not_allowed=此账号禁止创建组织 settings=组织设置 From 57781347b365c0dc88d234e021465ea2419fcb2b Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:30:33 +0800 Subject: [PATCH 199/206] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B8=B8=E5=AE=A2?= =?UTF-8?q?=E6=9F=A5=E7=9C=8B=E4=BA=91=E8=84=91=E5=88=97=E8=A1=A8=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 299133290..cbc9580f9 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -53,6 +53,9 @@ func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { } func CanCreateOrDebugJob(ctx *context.Context) bool { + if !ctx.IsSigned { + return false + } return ctx.Repo.CanWrite(models.UnitTypeCloudBrain) } From 1bf092295f01e200c1aaf1b9c769ac0a6c6f195c Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 8 Dec 2021 10:33:52 +0800 Subject: [PATCH 200/206] #999 fix bug --- models/dataset.go | 4 ++++ routers/user/profile.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/models/dataset.go b/models/dataset.go index 402a548ef..2b3de752b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -93,6 +93,7 @@ type SearchDatasetOptions struct { IncludePublic bool ListOptions SearchOrderBy + IsOwner bool } func CreateDataset(dataset *Dataset) (err error) { @@ -150,6 +151,9 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) + if !opts.IsOwner { + cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) + } } return cond diff --git a/routers/user/profile.go b/routers/user/profile.go index 0e29b6117..9da2bfa22 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -210,10 +210,15 @@ func Profile(ctx *context.Context) { total = int(count) case "datasets": + var isOwner = false + if ctx.User != nil && ctx.User.ID == ctxUser.ID { + isOwner = true + } datasetSearchOptions := &models.SearchDatasetOptions{ Keyword: keyword, OwnerID: ctxUser.ID, SearchOrderBy: orderBy, + IsOwner: isOwner, ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.ExplorePagingNum, From 4d938521cde74cda45e63fb2e2c2ee934e807cef Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:47:08 +0800 Subject: [PATCH 201/206] =?UTF-8?q?=E5=85=88=E7=94=A8=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E5=88=A4=E6=96=AD=E6=9D=83=E9=99=90=EF=BC=8C?= =?UTF-8?q?=E7=84=B6=E5=90=8E=E5=9C=A8=E7=94=A8=E8=BA=AB=E4=BB=BD=E5=88=A4?= =?UTF-8?q?=E6=96=AD=EF=BC=8C=E6=8F=90=E9=AB=98=E6=95=88=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index cbc9580f9..8a5e11997 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -2,6 +2,7 @@ package cloudbrain import ( "errors" + "strconv" "code.gitea.io/gitea/modules/setting" @@ -30,10 +31,13 @@ var ( ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) + log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) if err != nil { + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() } else { + log.Info("is job creator:" + strconv.FormatBool(ctx.User.ID == job.UserID)) return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID } @@ -44,11 +48,16 @@ func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { return false } + if !ctx.IsSigned { + return false + } return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { - + if !ctx.IsSigned { + return false + } return isAdminOrOwnerOrJobCreater(ctx, job, nil) } From ace4ec5ca6b8a3ff1e4667e92916ead67e31f6ad Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 10:59:31 +0800 Subject: [PATCH 202/206] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 8a5e11997..96dfce615 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -31,6 +31,9 @@ var ( ) func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + if !ctx.IsSigned { + return false + } log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) if err != nil { @@ -48,16 +51,12 @@ func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { return false } - if !ctx.IsSigned { - return false - } + return isAdminOrOwnerOrJobCreater(ctx, job, nil) } func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { - if !ctx.IsSigned { - return false - } + return isAdminOrOwnerOrJobCreater(ctx, job, nil) } @@ -74,7 +73,9 @@ func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { } func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { - + if !ctx.IsSigned { + return false + } if err != nil { return ctx.IsUserSiteAdmin() } else { @@ -125,7 +126,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, log.Error("no such resourceSpecId(%d)", resourceSpecId, ctx.Data["MsgID"]) return errors.New("no such resourceSpec") } - + jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -210,8 +211,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, + Type: models.TypeCloudBrainOne, + Uuid: uuid, }) if err != nil { From 19f9541eb74b4b52701d694ed03229fa4fc1f60a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 11:25:33 +0800 Subject: [PATCH 203/206] =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=9D=83=E9=99=90?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cloudbrain/cloudbrain.go | 11 +---------- routers/repo/cloudbrain.go | 2 +- routers/repo/modelarts.go | 4 ++-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 96dfce615..0f1c700d2 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -46,16 +46,7 @@ func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, er } -func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { - - if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { - return false - } - - return isAdminOrOwnerOrJobCreater(ctx, job, nil) -} - -func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { +func CanDeleteJob(ctx *context.Context, job *models.Cloudbrain) bool { return isAdminOrOwnerOrJobCreater(ctx, job, nil) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a3af588fb..ae7163d77 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -79,7 +79,7 @@ func CloudBrainIndex(ctx *context.Context) { ciTasks[i].CanDebug = false } - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f0b807d96..483efbc20 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -74,7 +74,7 @@ func NotebookIndex(ctx *context.Context) { } else { ciTasks[i].CanDebug = false } - ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -306,7 +306,7 @@ func TrainJobIndex(ctx *context.Context) { } for i, task := range tasks { - tasks[i].CanDel = cloudbrain.CanDeleteTrainJob(ctx, &task.Cloudbrain) + tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) } From 9714d66f86fbf1cefb46d2e587118d334b9d5e80 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 8 Dec 2021 14:26:36 +0800 Subject: [PATCH 204/206] fix 988 --- models/cloudbrain.go | 3 ++- modules/cron/tasks_basic.go | 6 +++--- modules/setting/setting.go | 8 +++++--- routers/repo/cloudbrain.go | 16 +++++++++++++++- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6e8ee1505..ceb552811 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1110,7 +1110,8 @@ func UpdateJob(job *Cloudbrain) error { func updateJob(e Engine, job *Cloudbrain) error { var sess *xorm.Session sess = e.Where("job_id = ?", job.JobID) - _, err := sess.Cols("status", "container_id", "container_ip").Update(job) + //_, err := sess.Cols("status", "container_id", "container_ip").Update(job) + _, err := sess.Update(job) return err } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 294690d45..b9838e66f 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -134,7 +134,7 @@ func registerHandleBlockChainUnSuccessRepos() { RegisterTaskFatal("handle_blockchain_unsuccess_repos", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessRepos() return nil @@ -145,7 +145,7 @@ func registerHandleBlockChainMergedPulls() { RegisterTaskFatal("handle_blockchain_merged_pull", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainMergedPulls() return nil @@ -156,7 +156,7 @@ func registerHandleBlockChainUnSuccessCommits() { RegisterTaskFatal("handle_blockchain_unsuccess_commits", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 3m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessCommits() return nil diff --git a/modules/setting/setting.go b/modules/setting/setting.go index f7b78235b..dd51623c1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -448,23 +448,24 @@ var ( GpuTypes string DebugServerHost string ResourceSpecs string + MaxDuration int64 //benchmark config IsBenchmarkEnabled bool - BenchmarkOwner string + BenchmarkOwner string BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetOwner string + Snn4imagenetOwner string Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreOwner string + BrainScoreOwner string BrainScoreName string BrainScoreServerHost string @@ -1238,6 +1239,7 @@ func NewContext() { JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug") GpuTypes = sec.Key("GPU_TYPES").MustString("") ResourceSpecs = sec.Key("RESOURCE_SPECS").MustString("") + MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index be5eab52d..04af5dba4 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -822,10 +822,24 @@ func SyncCloudbrainStatus() { taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) task.Status = taskRes.TaskStatuses[0].State if task.Status != string(models.JobWaiting) { + task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix() err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) - continue + } + + if task.Duration >= setting.MaxDuration { + log.Info("begin to stop job(%s), because of the duration", task.JobName) + err = cloudbrain.StopJob(task.JobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err) + continue + } + task.Status = string(models.JobStopped) + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } } } } From 00169d2d6b1cf894f55e73830dec16176417aa15 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 14:59:03 +0800 Subject: [PATCH 205/206] =?UTF-8?q?=E5=90=8E=E7=AB=AFcandebug=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E5=85=B3=E6=B3=A8=E4=BB=BB=E5=8A=A1=E7=8A=B6=E6=80=81?= =?UTF-8?q?=EF=BC=8C=E5=89=8D=E7=AB=AF=E6=A0=B9=E6=8D=AE=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E5=8F=98=E5=8C=96=E5=88=B7=E6=96=B0=E6=8C=89=E9=92=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index ae7163d77..8dcb405a2 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -71,14 +71,8 @@ func CloudBrainIndex(ctx *context.Context) { return } - timestamp := time.Now().Unix() for i, task := range ciTasks { - if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) { - ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) - } else { - ciTasks[i].CanDebug = false - } - + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) } From a355fc941594f96819322bf4547f38eee0ab2b0e Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 8 Dec 2021 15:01:08 +0800 Subject: [PATCH 206/206] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/modelarts.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 483efbc20..3994ba542 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -69,11 +69,8 @@ func NotebookIndex(ctx *context.Context) { } for i, task := range ciTasks { - if task.Status == string(models.JobRunning) { - ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) - } else { - ciTasks[i].CanDebug = false - } + + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) }