Merge pull request '#1523，用户分析增加模型转换数量， #1518，#3224，增加模型创建时查询训练任务接口，#3347，提供后端接口。' (#3432) from zouap into V20221228

Reviewed-on: https://openi.pcl.ac.cn/OpenI/aiforge/pulls/3432
Reviewed-by: ychao_1983 <ychao_1983@sina.com>

models/cloudbrain.go

@@ -205,7 +205,7 @@ type Cloudbrain struct {
 	BenchmarkTypeRankLink string      `xorm:"-"`
 	StartTime             timeutil.TimeStamp
 	EndTime               timeutil.TimeStamp
-	Cleared               bool   `xorm:"DEFAULT false"`
+	Cleared               bool           `xorm:"DEFAULT false"`
 	Spec                  *Specification `xorm:"-"`
 }
 
@@ -1823,7 +1823,7 @@ func QueryModelTrainJobVersionList(jobId string) ([]*Cloudbrain, int, error) {
 	return cloudbrains, int(len(cloudbrains)), nil
 }
 
-func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
+func QueryModelTrainJobList(repoId int64) ([]*Cloudbrain, int, error) {
 	sess := x.NewSession()
 	defer sess.Close()
 	var cond = builder.NewCond()
@@ -1840,14 +1840,14 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
 	// 	builder.In("type", 0, 1),
 	// )
 
-	cloudbrains := make([]*CloudbrainInfo, 0)
+	cloudbrains := make([]*Cloudbrain, 0)
 	if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC").
 		Find(&cloudbrains); err != nil {
 		return nil, 0, fmt.Errorf("Find: %v", err)
 	}
 
 	keys := make(map[string]string)
-	uniqueElements := make([]*CloudbrainInfo, 0)
+	uniqueElements := make([]*Cloudbrain, 0)
 	for _, entry := range cloudbrains {
 		if _, value := keys[entry.JobID]; !value {
 			keys[entry.JobID] = entry.DisplayJobName
@@ -1988,7 +1988,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) {
 	return getRepoCloudBrain(cb)
 }
 
-func IsCloudbrainExistByJobName(jobName string)(bool,error){
+func IsCloudbrainExistByJobName(jobName string) (bool, error) {
 	return x.Unscoped().Exist(&Cloudbrain{
 		JobName: jobName,
 	})
@@ -2172,25 +2172,25 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra
 		Limit(limit).
 		Find(&cloudbrains)
 }
+
 /**
   本方法考虑了再次调试的情况，多次调试取最后一次的任务的结束时间
- */
+*/
 func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
 	cloudbrains := make([]*Cloudbrain, 0, 10)
 	endTimeBefore := time.Now().Unix() - int64(days)*24*3600
 	missEndTimeBefore := endTimeBefore - 24*3600
-    sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
+	sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
         id, job_name, job_id,status,end_time,updated_unix,cleared
         FROM   cloudbrain
         where type=0 and job_type='DEBUG'
         ORDER  BY job_name, updated_unix DESC) a
         where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false`
 
-	return cloudbrains, x.Unscoped().SQL(sql,missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)
+	return cloudbrains, x.Unscoped().SQL(sql, missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)
 
 }
 
-
 func UpdateCloudBrainRecordsCleared(ids []int64) error {
 	pageSize := 150
 	n := len(ids) / pageSize

modules/convert/cloudbrain.go

@@ -34,8 +34,7 @@ func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain {
 
 		StartTime: int64(task.StartTime),
 		EndTime:   int64(task.EndTime),
-
-		Spec: ToSpecification(task.Spec),
+		Spec:      ToSpecification(task.Spec),
 	}
 }
 func ToAttachment(attachment *models.Attachment) *api.AttachmentShow {
@@ -89,6 +88,9 @@ func ToDataset(dataset *models.Dataset) *api.Dataset {
 }
 
 func ToSpecification(s *models.Specification) *api.SpecificationShow {
+	if s == nil {
+		return nil
+	}
 	return &api.SpecificationShow{
 		ID:              s.ID,
 		AccCardsNum:     s.AccCardsNum,

options/locale/locale_en-US.ini

@@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count
 static.CloudBrainRunTime=CloudBrain Run Time
 static.CommitDatasetNum=Commit Dataset Count
 static.CommitModelCount=Commit Model Count
+static.ModelConvertCount=Model Convert Count
 static.UserIndex=Normalized user index
 static.UserIndexPrimitive=User Index
 static.countdate=Count Date

options/locale/locale_zh-CN.ini

@@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数
 static.CloudBrainRunTime=云脑运行时间（小时）
 static.CommitDatasetNum=上传（提交）数据集文件数
 static.CommitModelCount=提交模型数
+static.ModelConvertCount=模型转换数
 static.UserIndex=归一化用户指数
 static.UserIndexPrimitive=用户指数
 static.countdate=系统统计时间
@@ -1672,7 +1673,7 @@ issues.action_assignee_no_select=未指派
 issues.opened_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s创建
 pulls.merged_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s 合并
 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并
-issues.closed_by=按 <a href="%[2]s">%[3]s</a> 关闭%[1]s
+issues.closed_by=由 <a href="%[2]s">%[3]s</a> 创建，被关闭于 %[1]s
 issues.opened_by_fake=由 %[2]s 于 %[1]s创建
 issues.closed_by_fake=通过 %[2]s 关闭 %[1]s
 issues.previous=上一页

routers/api/v1/api.go

@@ -1024,6 +1024,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Get("/query_model_byName", repo.QueryModelByName)
 					m.Get("/query_model_for_predict", repo.QueryModelListForPredict)
 					m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict)
+					m.Get("/query_train_job", repo.QueryTrainJobList)
+					m.Get("/query_train_job_version", repo.QueryTrainJobVersionList)
 					m.Get("/query_train_model", repo.QueryTrainModelList)
 					m.Post("/create_model_convert", repo.CreateModelConvert)
 					m.Post("/convert_stop", repo.StopModelConvert)

routers/api/v1/repo/modelmanage.go

@@ -4,8 +4,10 @@ import (
 	"net/http"
 
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
+	api "code.gitea.io/gitea/modules/structs"
 	routerRepo "code.gitea.io/gitea/routers/repo"
 )
 
@@ -54,6 +56,21 @@ func QueryModelListForPredict(ctx *context.APIContext) {
 	routerRepo.QueryModelListForPredict(ctx.Context)
 }
 
+func QueryTrainJobList(ctx *context.APIContext) {
+	result, err := routerRepo.QueryTrainJobListApi(ctx.Context)
+	if err != nil {
+		log.Info("query error." + err.Error())
+		ctx.JSON(http.StatusOK, nil)
+	} else {
+		re := make([]*api.Cloudbrain, 0)
+		for _, task := range result {
+			conRe := convert.ToCloudBrain(task)
+			re = append(re, conRe)
+		}
+		ctx.JSON(http.StatusOK, re)
+	}
+}
+
 func QueryTrainModelList(ctx *context.APIContext) {
 	result, err := routerRepo.QueryTrainModelFileById(ctx.Context)
 	if err != nil {
@@ -63,6 +80,21 @@ func QueryTrainModelList(ctx *context.APIContext) {
 	ctx.JSON(http.StatusOK, re)
 }
 
+func QueryTrainJobVersionList(ctx *context.APIContext) {
+	result, err := routerRepo.QueryTrainJobVersionListApi(ctx.Context)
+	if err != nil {
+		log.Info("query error." + err.Error())
+		ctx.JSON(http.StatusOK, nil)
+	} else {
+		re := make([]*api.Cloudbrain, 0)
+		for _, task := range result {
+			conRe := convert.ToCloudBrain(task)
+			re = append(re, conRe)
+		}
+		ctx.JSON(http.StatusOK, re)
+	}
+}
+
 func convertFileFormat(result []storage.FileInfo) []FileInfo {
 	re := make([]FileInfo, 0)
 	if result != nil {

routers/repo/ai_model_manage.go

@@ -2,7 +2,6 @@ package repo
 
 import (
 	"archive/zip"
-	"code.gitea.io/gitea/services/repository"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -12,6 +11,8 @@ import (
 	"regexp"
 	"strings"
 
+	"code.gitea.io/gitea/services/repository"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
@@ -710,36 +711,42 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con
 }
 
 func QueryTrainJobVersionList(ctx *context.Context) {
+
+	VersionListTasks, err := QueryTrainJobVersionListApi(ctx)
+	if err != nil {
+		ctx.JSON(200, nil)
+	} else {
+		ctx.JSON(200, VersionListTasks)
+	}
+}
+
+func QueryTrainJobVersionListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
 	log.Info("query train job version list. start.")
 	JobID := ctx.Query("jobId")
 	if JobID == "" {
 		JobID = ctx.Query("JobId")
 	}
-
 	VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID)
-
 	log.Info("query return count=" + fmt.Sprint(count))
 
+	return VersionListTasks, err
+}
+
+func QueryTrainJobList(ctx *context.Context) {
+	VersionListTasks, err := QueryTrainJobListApi(ctx)
 	if err != nil {
-		ctx.ServerError("QueryTrainJobList:", err)
+		ctx.JSON(200, nil)
 	} else {
 		ctx.JSON(200, VersionListTasks)
 	}
 }
 
-func QueryTrainJobList(ctx *context.Context) {
-	log.Info("query train job list. start.")
+func QueryTrainJobListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
 	repoId := ctx.QueryInt64("repoId")
-
 	VersionListTasks, count, err := models.QueryModelTrainJobList(repoId)
 	log.Info("query return count=" + fmt.Sprint(count))
 
-	if err != nil {
-		ctx.ServerError("QueryTrainJobList:", err)
-	} else {
-		ctx.JSON(200, VersionListTasks)
-	}
-
+	return VersionListTasks, err
 }
 
 func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) {

routers/repo/cloudbrain.go

@@ -1638,6 +1638,21 @@ func CloudBrainDownloadModel(ctx *context.Context) {
 	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
+
+func CloudBrainDownloadMultiModel(ctx *context.Context) {
+	parentDir := ctx.Query("parentDir")
+	jobName := ctx.Query("jobName")
+	filePath := "jobs/" + jobName + "/model/" + parentDir
+	allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
+	if err == nil {
+		returnFileName := jobName + ".zip"
+		MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
+	} else {
+		log.Info("error,msg=" + err.Error())
+		ctx.ServerError("no file to download.", err)
+	}
+}
+
 func CloudBrainDownloadInferenceResult(ctx *context.Context) {
 	parentDir := ctx.Query("parentDir")
 	fileName := ctx.Query("fileName")

routers/repo/modelarts.go

@@ -2653,6 +2653,44 @@ func InferenceJobShow(ctx *context.Context) {
 	ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow)
 }
 
+func MultiModelDownload(ctx *context.Context) {
+	var (
+		err error
+	)
+	jobID := ctx.Params(":jobid")
+	versionName := ctx.Query("version_name")
+	parentDir := ctx.Query("parent_dir")
+
+	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
+	if err != nil {
+		log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error())
+		return
+	}
+
+	if task.ComputeResource == models.NPUResource {
+		path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/")
+		allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
+		if err == nil {
+			returnFileName := task.DisplayJobName + ".zip"
+			ObsDownloadManyFile(path, ctx, returnFileName, allFile)
+		} else {
+			log.Info("error,msg=" + err.Error())
+			ctx.ServerError("no file to download.", err)
+		}
+	} else if task.ComputeResource == models.GPUResource {
+		filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir
+		allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
+		if err == nil {
+			returnFileName := task.DisplayJobName + ".zip"
+			MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
+		} else {
+			log.Info("error,msg=" + err.Error())
+			ctx.ServerError("no file to download.", err)
+		}
+	}
+
+}
+
 func ModelDownload(ctx *context.Context) {
 	var (
 		err error

routers/routes/routes.go

@@ -1186,6 +1186,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate)
 				m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
 				m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel)
+				m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 			})
 			m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew)
 			m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
@@ -1209,6 +1210,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel)
 					//m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
 					m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel)
+					m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 					//m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir)
 					//m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
 					m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew)
@@ -1221,7 +1223,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Group("/:jobid", func() {
 					m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow)
 					m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult)
-
+					m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
 					m.Get("/downloadall", repo.DownloadInferenceResultFile)
 				})
 				m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew)