Merge remote-tracking branch 'origin/V20211101' into zouap

modules/modelarts/resty.go

@@ -366,6 +366,10 @@ sendjob:
 			return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
 		}
 		log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+		if res.StatusCode() == 400 {
+			temp.ErrorCode = "0404"
+			temp.ErrorMsg = "启动文件未找到！"
+		}
 		return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
 	}
 

modules/setting/setting.go

@@ -1295,7 +1295,7 @@ func SetRadarMapConfig() {
 
 	RadarMap.Impact = sec.Key("impact").MustFloat64(0.3)
 	RadarMap.ImpactWatch = sec.Key("impact_watch").MustFloat64(0.1)
-	RadarMap.ImpactStar = sec.Key("impact_star").MustFloat64(0.3)
+	RadarMap.ImpactStar = sec.Key("impact_star").MustFloat64(0.2)
 	RadarMap.ImpactFork = sec.Key("impact_fork").MustFloat64(0.3)
 	RadarMap.ImpactCodeDownload = sec.Key("impact_code_download").MustFloat64(0.2)
 	RadarMap.ImpactComments = sec.Key("impact_comments").MustFloat64(0.1)

modules/storage/obs.go

@@ -5,7 +5,6 @@
 package storage
 
 import (
-	"fmt"
 	"io"
 	"path"
 	"strconv"
@@ -18,6 +17,15 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 )
 
+type FileInfo struct {
+	FileName string `json:"FileName"`
+	ModTime  string `json:"ModTime"`
+	IsDir    bool   `json:"IsDir"`
+	Size     int64  `json:"Size"`
+	ParenDir string `json:"ParenDir"`
+	UUID     string `json:"UUID"`
+}
+
 //check if has the object
 //todo：修改查询方式
 func ObsHasObject(path string) (bool, error) {
@@ -141,8 +149,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
 			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
 		return output.Body, nil
 	} else if obsError, ok := err.(obs.ObsError); ok {
-		fmt.Printf("Code:%s\n", obsError.Code)
-		fmt.Printf("Message:%s\n", obsError.Message)
+		log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
 		return nil, obsError
 	} else {
 		return nil, err
@@ -160,40 +167,49 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
 			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
 		return output.Body, nil
 	} else if obsError, ok := err.(obs.ObsError); ok {
-		fmt.Printf("Code:%s\n", obsError.Code)
-		fmt.Printf("Message:%s\n", obsError.Message)
+		log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
 		return nil, obsError
 	} else {
 		return nil, err
 	}
 }
 
-func GetObsListObject(jobName string) ([]string, error) {
-	// jobName = "liuzx202110271830856"
+func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = setting.Bucket
-	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath), "/")
-	log.Info("input.Prefix:", input.Prefix)
+	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
 	output, err := ObsCli.ListObjects(input)
-	log.Info("output.Prefix:", output)
-	ModelListArr := make([]string, 0)
+	fileInfos := make([]FileInfo, 0)
 	if err == nil {
-		fmt.Printf("RequestId:%s\n", output.RequestId)
-		for index, val := range output.Contents {
-			fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n",
-				index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size)
+		for _, val := range output.Contents {
 			str1 := strings.Split(val.Key, "/")
-			ModelList := str1[len(str1)-1]
-			ModelListArr = append(ModelListArr, ModelList)
-			log.Info("ModelListArr.Prefix:", ModelListArr)
+			var isDir bool
+			var fileName,nextParentDir string
+			if strings.HasSuffix(val.Key, "/") {
+				fileName = str1[len(str1)-2]
+				isDir = true
+				nextParentDir = fileName
+				if fileName == parentDir || (fileName + "/") == setting.OutPutPath {
+					continue
+				}
+			} else {
+				fileName = str1[len(str1)-1]
+				isDir = false
+			}
+
+			fileInfo := FileInfo{
+				ModTime: val.LastModified.Format("2006-01-02 15:04:05"),
+				FileName: fileName,
+				Size: val.Size,
+				IsDir:isDir,
+				ParenDir: nextParentDir,
+			}
+			fileInfos = append(fileInfos, fileInfo)
 		}
-		return ModelListArr, err
+		return fileInfos, err
 	} else {
 		if obsError, ok := err.(obs.ObsError); ok {
-			fmt.Println(obsError.Code)
-			fmt.Println(obsError.Message)
-		} else {
-			fmt.Println(err)
+			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
 		}
 		return nil, err
 	}
@@ -222,20 +238,17 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file
 	return output.SignedUrl, nil
 }
 
-func GetObsCreateSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {
-
+func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) {
 	input := &obs.CreateSignedUrlInput{}
 	input.Bucket = setting.Bucket
-	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
+	input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")
+	
 	input.Expires = 60 * 60
-	input.Method = obs.HttpMethodPut
-
-	input.QueryParams = map[string]string{
-		"partNumber": com.ToStr(partNumber, 10),
-		"uploadId":   uploadId,
-		//"partSize":	com.ToStr(partSize,10),
-	}
+	input.Method = obs.HttpMethodGet
 
+	reqParams := make(map[string]string)
+	reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\""
+	input.QueryParams = reqParams
 	output, err := ObsCli.CreateSignedUrl(input)
 	if err != nil {
 		log.Error("CreateSignedUrl failed:", err.Error())

routers/repo/cloudbrain.go

@@ -510,7 +510,7 @@ func CloudBrainShowModels(ctx *context.Context) {
 		return
 	}
 
-	var fileInfos []FileInfo
+	var fileInfos []storage.FileInfo
 	err = json.Unmarshal([]byte(dirs), &fileInfos)
 	if err != nil {
 		log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
@@ -583,74 +583,6 @@ func CloudBrainDownloadModel(ctx *context.Context) {
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
 
-// func TrainJobloadModel(ctx *context.Context) {
-// 	parentDir := ctx.Query("parentDir")
-// 	fileName := ctx.Query("fileName")
-// 	jobName := ctx.Query("jobName")
-// 	filePath := "jobs/" + jobName + "/model/" + parentDir
-// 	url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
-// 	if err != nil {
-// 		log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
-// 		ctx.ServerError("PresignedGetURL", err)
-// 		return
-// 	}
-
-// 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
-// }
-
-func TrainJobListModel(ctx *context.Context) {
-	ctx.Data["PageIsCloudBrain"] = true
-	jobID := ctx.Params(":jobid")
-	task, err := models.GetCloudbrainByJobID(jobID)
-	if err != nil {
-		log.Error("no such job!", ctx.Data["err"])
-		ctx.ServerError("no such job:", err)
-		return
-	}
-	TrainJobListModel, err := storage.GetObsListObject(task.JobName)
-	log.Info("TrainJobListModel", TrainJobListModel)
-	fmt.Println("TrainJobListModel:", TrainJobListModel)
-	if err != nil {
-		log.Info("get TrainJobListModel failed:", err)
-		return
-	}
-	ctx.Data["task"] = task
-	ctx.Data["JobID"] = jobID
-	ctx.Data["ListModel"] = TrainJobListModel
-	ctx.HTML(200, tplModelArtsTrainJobListModel)
-}
-
-func TrainJobDownloadModel(ctx *context.Context) {
-	JobName := ctx.Query("JobName")
-	fileName := ctx.Query("file_name")
-
-	// JobName = "liuzx202110271830856"
-	// fileName = "Untitled.ipynb"
-
-	body, err := storage.ObsModelDownload(JobName, fileName)
-	if err != nil {
-		log.Info("download error.")
-	} else {
-		defer body.Close()
-		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
-		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
-		p := make([]byte, 1024)
-		var readErr error
-		var readCount int
-		// 读取对象内容
-		for {
-			readCount, readErr = body.Read(p)
-			if readCount > 0 {
-				ctx.Resp.Write(p[:readCount])
-				//fmt.Printf("%s", p[:readCount])
-			}
-			if readErr != nil {
-				break
-			}
-		}
-	}
-}
-
 func GetRate(ctx *context.Context) {
 	var jobID = ctx.Params(":jobid")
 	job, err := models.GetCloudbrainByJobID(jobID)

routers/repo/dir.go

@@ -21,15 +21,6 @@ const (
 	tplDirIndex base.TplName = "repo/datasets/dirs/index"
 )
 
-type FileInfo struct {
-	FileName string `json:"FileName"`
-	ModTime  string `json:"ModTime"`
-	IsDir    bool   `json:"IsDir"`
-	Size     int64  `json:"Size"`
-	ParenDir string `json:"ParenDir"`
-	UUID     string `json:"UUID"`
-}
-
 type RespGetDirs struct {
 	ResultCode string `json:"resultCode"`
 	FileInfos  string `json:"fileInfos"`
@@ -59,7 +50,7 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) {
 			return
 		}
 
-		var fileInfos []FileInfo
+		var fileInfos []storage.FileInfo
 		err = json.Unmarshal([]byte(dirs), &fileInfos)
 		if err != nil {
 			log.Error("json.Unmarshal failed:", err.Error())

routers/repo/modelarts.go

@@ -3,7 +3,6 @@ package repo
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -12,18 +11,18 @@ import (
 	"strings"
 	"time"
 
-	"code.gitea.io/gitea/modules/git"
-	"code.gitea.io/gitea/modules/modelarts"
-	"code.gitea.io/gitea/modules/obs"
-	"code.gitea.io/gitea/modules/storage"
-	"github.com/unknwon/com"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/modelarts"
+	"code.gitea.io/gitea/modules/obs"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+
+	"github.com/unknwon/com"
 )
 
 const (
@@ -39,7 +38,7 @@ const (
 	tplModelArtsTrainJobIndex     base.TplName = "repo/modelarts/trainjob/index"
 	tplModelArtsTrainJobNew       base.TplName = "repo/modelarts/trainjob/new"
 	tplModelArtsTrainJobShow      base.TplName = "repo/modelarts/trainjob/show"
-	tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model"
+	tplModelArtsTrainJobShowModels		  base.TplName = "repo/modelarts/trainjob/models/index"
 )
 
 // MustEnableDataset check if repository enable internal cb
@@ -492,13 +491,13 @@ func NotebookDel(ctx *context.Context) {
 func TrainJobIndex(ctx *context.Context) {
 	MustEnableModelArts(ctx)
 
-	can, err := canUserCreateTrainJob(ctx.User.ID)
-	if err != nil {
-		ctx.ServerError("canUserCreateTrainJob", err)
-		return
-	}
-
-	ctx.Data["CanCreate"] = can
+	//can, err := canUserCreateTrainJob(ctx.User.ID)
+	//if err != nil {
+	//	ctx.ServerError("canUserCreateTrainJob", err)
+	//	return
+	//}
+	//
+	//ctx.Data["CanCreate"] = can
 
 	repo := ctx.Repo.Repository
 	page := ctx.QueryInt("page")
@@ -531,17 +530,17 @@ func TrainJobIndex(ctx *context.Context) {
 func TrainJobNew(ctx *context.Context) {
 	ctx.Data["PageIsCloudBrain"] = true
 
-	can, err := canUserCreateTrainJob(ctx.User.ID)
-	if err != nil {
-		ctx.ServerError("canUserCreateTrainJob", err)
-		return
-	}
-
-	if !can {
-		log.Error("the user can not create train-job")
-		ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job"))
-		return
-	}
+	//can, err := canUserCreateTrainJob(ctx.User.ID)
+	//if err != nil {
+	//	ctx.ServerError("canUserCreateTrainJob", err)
+	//	return
+	//}
+	//
+	//if !can {
+	//	log.Error("the user can not create train-job")
+	//	ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job"))
+	//	return
+	//}
 
 	t := time.Now()
 	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
@@ -1060,3 +1059,43 @@ func getConfigList(perPage, page int, sortBy, order, searchContent, configType s
 
 	return list, nil
 }
+
+func TrainJobShowModels(ctx *context.Context) {
+	ctx.Data["PageIsCloudBrain"] = true
+
+	jobID := ctx.Params(":jobid")
+	parentDir := ctx.Query("parentDir")
+	dirArray := strings.Split(parentDir, "/")
+	task, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		log.Error("no such job!", ctx.Data["msgID"])
+		ctx.ServerError("no such job:", err)
+		return
+	}
+
+	models, err := storage.GetObsListObject(task.JobName, parentDir)
+	if err != nil {
+		log.Info("get TrainJobListModel failed:", err)
+		ctx.ServerError("GetObsListObject:", err)
+		return
+	}
+
+	ctx.Data["Path"] = dirArray
+	ctx.Data["Dirs"] = models
+	ctx.Data["task"] = task
+	ctx.Data["JobID"] = jobID
+	ctx.HTML(200, tplModelArtsTrainJobShowModels)
+}
+
+func TrainJobDownloadModel(ctx *context.Context) {
+	parentDir := ctx.Query("parentDir")
+	fileName := ctx.Query("fileName")
+	jobName := ctx.Query("jobName")
+	url, err := storage.GetObsCreateSignedUrl(jobName, parentDir, fileName)
+	if err != nil {
+		log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
+		ctx.ServerError("GetObsCreateSignedUrl", err)
+		return
+	}
+	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
+}

routers/routes/routes.go

@@ -991,7 +991,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
 					m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
 					m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog)
-					m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel)
+					m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobShowModels)
 					m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel)
 				})
 				m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew)

templates/repo/modelarts/notebook/index.tmpl

@@ -423,7 +423,7 @@
 
     // 加载任务状态
     var timeid = window.setInterval(loadJobStatus, 15000);
-     $(document).ready(loadJobStatus);
+    $(document).ready(loadJobStatus);
     function loadJobStatus() {
         $(".job-status").each((index, job) => {
             const jobID = job.dataset.jobid;
@@ -435,7 +435,7 @@
             $.get(`/api/v1/repos/${repoPath}/modelarts/notebook/${jobID}`, (data) => {
                 const jobID = data.JobID
                 const status = data.JobStatus
-                if (status != job.textContent.trim() || status=='RUNNING') {
+                if (status != job.textContent.trim()) {
                     //$('#' + jobID).text(status)
                     //if (status == 'STOPPED') {
                         window.location.reload()

templates/repo/modelarts/trainjob/index.tmpl

@@ -459,7 +459,7 @@
 
     // 加载任务状态
     var timeid = window.setInterval(loadJobStatus, 15000);
-     $(document).ready(loadJobStatus);
+    $(document).ready(loadJobStatus);
     function loadJobStatus() {
         $(".job-status").each((index, job) => {
             const jobID = job.dataset.jobid;
@@ -473,7 +473,7 @@
             $.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => {
                 const jobID = data.JobID
                 const status = data.JobStatus
-                if (status != job.textContent.trim() || status=='RUNNING') {
+                if (status != job.textContent.trim()) {
                     //$('#' + jobID).text(status)
                     //if (status == 'STOPPED') {
                         window.location.reload()

templates/repo/modelarts/trainjob/models/dir_list.tmpl

@@ -0,0 +1,27 @@
+{{if .Dirs}}
+    <table id="repo-files-table" class="ui single line table">
+      <tbody>
+        {{range .Dirs}}
+        <tr>
+          <td class="name four wide">
+            <span class="truncate">
+                <span class="octicon octicon-file-directory"></span>
+                <a class="title" href="{{if .IsDir}}{{$.RepoLink}}/modelarts/train-job/{{$.JobID}}/models?parentDir={{.ParenDir}}{{else}}{{$.RepoLink}}/modelarts/train-job/{{$.JobID}}/download_model?parentDir={{.ParenDir}}&fileName={{.FileName}}&jobName={{$.task.JobName}}{{end}}">
+                  <span class="fitted">{{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}}</span> {{.FileName}}
+                </a>
+            </span>
+          </td>
+          <td class="message nine wide">
+            <span class="truncate has-emoji">
+              {{.Size | FileSize}}
+            </span>
+          </td>
+          <td class="text right age three wide">
+            <span class="time-since poping up">{{.ModTime}}</span>
+          </td>
+        </tr>
+        {{end}}
+      </tbody>
+    </table>
+
+{{end}}

templates/repo/modelarts/trainjob/models/index.tmpl

@@ -0,0 +1,29 @@
+{{template "base/head" .}}
+<div class="repository dataset dir-list view">
+	{{template "repo/header" .}}
+	<form class="ui container">
+    <div class="ui stackable grid {{if .Error}}hide{{end}}" id="dir-content">
+      <div class="row">
+        <div class="column sixteen wide">
+          <p>
+            {{ range $index, $item := .Path }}<a href='{{$.Link}}/?parentDir={{if gt $index 0}}{{DatasetPathJoin $.Path $index "/"}}{{else}}{{end}}'>{{ $item }}</a><span class="directory-seperator">/</span>{{ end }}
+          </p>
+        </div>
+      </div>
+    </div>
+
+    <div class="ui grid">
+      <div class="row">
+        <div class="ui sixteen wide column">
+          <div class="dir list">
+            {{template "repo/modelarts/trainjob/models/dir_list" .}}
+          </div>
+        </div>
+      </div>
+    </div>
+  </form>
+</div>
+
+
+
+{{template "base/footer" .}}