%[2]s at %s#%[2]s`
reject_pull_request = `suggested changes for %s#%[2]s`
upload_dataset=`upload dataset %s`
-task_gpudebugjob=`created CPU/GPU type debugging task%s`
+task_gpudebugjob=`created CPU/GPU type debugging task %s`
task_npudebugjob=`created NPU type debugging task %s`
-task_nputrainjob=`created NPU training task%s`
+task_nputrainjob=`created NPU training task %s`
task_inferencejob=`created reasoning task %s`
task_benchmark=`created profiling task %s`
task_createmodel=`created new model %s`
-task_gputrainjob=`created CPU/GPU training task%s`
-task_c2netnputrainjob=`created NPU training task%s`
-task_c2netgputrainjob=`created CPU/GPU training task%s`
+task_gputrainjob=`created CPU/GPU training task %s`
+task_c2netnputrainjob=`created NPU training task %s`
+task_c2netgputrainjob=`created CPU/GPU training task %s`
[tool]
ago = %s ago
@@ -3092,6 +3093,8 @@ all_ai_center=All Computing NET
resource_cluster = Resource Cluster
resource_cluster_openi = OpenI Resource Cluster
resource_cluster_c2net = China Computing NET
+resource_cluster_openi_simple = OpenI
+resource_cluster_c2net_simple = Computing NET
compute_resource = Computing resources
task_name = Task name
task_type = Task type
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index 39edb2da5..4c38aefcc 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -1110,6 +1110,7 @@ modelarts.deletetime=删除时间
modelarts.version_nums=版本数
modelarts.version=版本
modelarts.computing_resources=计算资源
+modelarts.cluster.computing_resources=集群/计算资源
modelarts.ai_center=智算中心
modelarts.card_type=卡类型
modelarts.cluster=集群
@@ -3106,6 +3107,8 @@ all_ai_center=全部智算中心
resource_cluster = 算力集群
resource_cluster_openi = 启智集群
resource_cluster_c2net = 智算网络集群
+resource_cluster_openi_simple = 启智
+resource_cluster_c2net_simple = 智算网络
compute_resource = 计算资源
task_name = 任务名称
task_type = 任务类型
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index aa51c6e1a..0b941b400 100755
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -916,6 +916,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/cloudbrain", func() {
m.Get("/:id", repo.GetCloudbrainTask)
m.Get("/:id/log", repo.CloudbrainGetLog)
+ m.Get("/:id/download_log_file", repo.CloudbrainDownloadLogFile)
m.Group("/train-job", func() {
m.Group("/:jobid", func() {
m.Get("", repo.GetModelArtsTrainJobVersion)
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index c3a803f70..a8c5762e4 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -6,13 +6,17 @@
package repo
import (
- "code.gitea.io/gitea/modules/notification"
+ "bufio"
"encoding/json"
+ "io"
"net/http"
+ "os"
"sort"
"strings"
"time"
+ "code.gitea.io/gitea/modules/notification"
+
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/models"
@@ -366,7 +370,7 @@ func CloudbrainForModelConvertGetLog(ctx *context.Context) {
ctx.JSON(http.StatusOK, result)
}
-func CloudbrainGetLog(ctx *context.Context) {
+func CloudbrainDownloadLogFile(ctx *context.Context) {
ID := ctx.Params(":id")
job, err := models.GetCloudbrainByID(ID)
if err != nil {
@@ -374,14 +378,124 @@ func CloudbrainGetLog(ctx *context.Context) {
ctx.ServerError(err.Error(), err)
return
}
+ prefix := "/" + setting.CBCodePathPrefix + job.JobName + "/model"
+ files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
+ if err != nil {
+ log.Error("query cloudbrain model failed: %v", err)
+ return
+ }
+ fileName := ""
+ for _, file := range files {
+ if strings.HasSuffix(file.FileName, "log.txt") {
+ fileName = file.FileName
+ break
+ }
+ }
+ if fileName != "" {
+ url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName)
+ if err != nil {
+ log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
+ ctx.ServerError("Get minio get SignedUrl failed", err)
+ return
+ }
+ http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
+ }
+}
- result := CloudbrainGetLogByJobId(job.JobID, job.JobName)
+func CloudbrainGetLog(ctx *context.Context) {
+ ID := ctx.Params(":id")
+ startLine := ctx.QueryInt("base_line")
+ lines := ctx.QueryInt("lines")
+ endLine := startLine + lines
+ order := ctx.Query("order")
+ if order == "asc" {
+ endLine = startLine
+ startLine = endLine - lines
+ if startLine < 0 {
+ startLine = 0
+ }
+ }
+ job, err := models.GetCloudbrainByID(ID)
+ if err != nil {
+ log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
+ ctx.ServerError(err.Error(), err)
+ return
+ }
+ result := getLogFromModelDir(job.JobName, startLine, endLine)
if result == nil {
log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
ctx.ServerError(err.Error(), err)
return
}
- ctx.JSON(http.StatusOK, result)
+
+ re := map[string]interface{}{
+ "JobID": ID,
+ "LogFileName": result["FileName"],
+ "StartLine": startLine,
+ "EndLine": result["endLine"],
+ "Content": result["Content"],
+ "Lines": result["lines"],
+ "CanLogDownload": result["FileName"] != "",
+ }
+ //result := CloudbrainGetLogByJobId(job.JobID, job.JobName)
+
+ ctx.JSON(http.StatusOK, re)
+}
+
+func getLogFromModelDir(jobName string, startLine int, endLine int) map[string]interface{} {
+ prefix := "/" + setting.CBCodePathPrefix + jobName + "/model"
+ files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
+ if err != nil {
+ log.Error("query cloudbrain model failed: %v", err)
+ return nil
+ }
+
+ re := ""
+ fileName := ""
+ count := 0
+ fileEndLine := endLine
+ for _, file := range files {
+ if strings.HasSuffix(file.FileName, "log.txt") {
+ fileName = file.FileName
+ path := storage.GetMinioPath(jobName+"/model/", file.FileName)
+ log.Info("path=" + path)
+ reader, err := os.Open(path)
+ defer reader.Close()
+ if err == nil {
+ r := bufio.NewReader(reader)
+ for i := 0; i < endLine; i++ {
+ line, error := r.ReadString('\n')
+ log.Info("line=" + line)
+ fileEndLine = i
+ if error == io.EOF {
+ log.Info("read file completed.")
+ break
+ }
+ if error != nil {
+ log.Info("read file error." + error.Error())
+ break
+ }
+ if error == nil {
+ if i >= startLine {
+ re = re + line
+ count++
+ }
+ }
+ }
+ } else {
+ log.Info("error:" + err.Error())
+ }
+ break
+ }
+ }
+
+ return map[string]interface{}{
+ "JobName": jobName,
+ "Content": re,
+ "FileName": fileName,
+ "lines": count,
+ "endLine": fileEndLine,
+ }
}
func CloudBrainModelConvertList(ctx *context.APIContext) {
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 3294efdd1..419c6d6a4 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -154,6 +154,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
if job.StartTime == 0 && result.JobInfo.StartedAt > 0 {
job.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
+ oldStatus := job.Status
job.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
job.Duration = result.JobInfo.RunSec
job.TrainJobDuration = models.ConvertDurationToStr(job.Duration)
@@ -176,6 +177,9 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
aiCenterName = temp[1]
}
}
+ if oldStatus != job.Status {
+ notification.NotifyChangeCloudbrainStatus(job, oldStatus)
+ }
err = models.UpdateTrainJobVersion(job)
if err != nil {
log.Error("UpdateJob failed:", err)
diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go
index cdde7596c..abc2027d3 100755
--- a/routers/repo/grampus.go
+++ b/routers/repo/grampus.go
@@ -767,10 +767,6 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
command += commandDownload
}
- //check download result
- commandCheckRes := "bash -c \"[[ $? -eq 0 ]] && exit 0 || exit -1;\";"
- command += commandCheckRes
-
//unzip code & dataset
toolUnzip := "unzip -q '"
if strings.HasSuffix(datasetName, ".tar.gz") {
@@ -779,16 +775,22 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + toolUnzip + datasetName + "';"
command += commandUnzip
- //check unzip result
- commandCheckRes = "bash -c \"[[ $? -eq 0 ]] && exit 0 || exit -1;\";"
- command += commandCheckRes
-
command += "echo \"unzip finished;start to exec code;\";"
+ // set export
+ var commandExport string
+ if processorType == grampus.ProcessorTypeNPU {
+ commandExport = "export bucket=" + setting.Bucket + " && export remote_path=" + outputRemotePath + ";"
+ } else if processorType == grampus.ProcessorTypeGPU {
+ commandExport = "export env=" + setting.Grampus.Env + " && export remote_path=" + outputRemotePath + ";"
+ }
+
+ command += commandExport
+
//exec code
var parameters models.Parameters
var paramCode string
- param := make([]models.Parameter, 0)
+
if len(paramSrc) != 0 {
err := json.Unmarshal([]byte(paramSrc), ¶meters)
if err != nil {
@@ -797,10 +799,6 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
}
for _, parameter := range parameters.Parameter {
- param = append(param, models.Parameter{
- Label: parameter.Label,
- Value: parameter.Value,
- })
paramCode += " --" + parameter.Label + "=" + parameter.Value
}
}
@@ -820,15 +818,15 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
//upload models
if processorType == grampus.ProcessorTypeNPU {
- commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;"
+ commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_npu " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;"
command += commandUpload
} else if processorType == grampus.ProcessorTypeGPU {
- commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;"
+ commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_gpu " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;"
command += commandUpload
}
//check exec result
- commandCheckRes = "bash -c \"[[ $result -eq 0 ]] && exit 0 || exit -1\""
+ commandCheckRes := "bash -c \"[[ $result -eq 0 ]] && exit 0 || exit -1\""
command += commandCheckRes
return command, nil
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 763308930..697e6591e 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -896,12 +896,12 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
ctx.Data["display_job_name"] = task.DisplayJobName
ctx.Data["job_name"] = task.JobName
- attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID)
- if err != nil {
- ctx.ServerError("GetAllUserAttachments failed:", err)
- return err
- }
- ctx.Data["attachments"] = attachs
+ // attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID)
+ // if err != nil {
+ // ctx.ServerError("GetAllUserAttachments failed:", err)
+ // return err
+ // }
+ // ctx.Data["attachments"] = attachs
var resourcePools modelarts.ResourcePool
if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil {
@@ -945,12 +945,16 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
ctx.ServerError("GetBranches error:", err)
return err
}
-
+ _, _, datasetNames, _, err := getDatasUrlListByUUIDS(task.Uuid)
+ if err != nil {
+ ctx.ServerError("GetAllUserAttachments failed:", err)
+ return err
+ }
ctx.Data["branches"] = branches
ctx.Data["branch_name"] = task.BranchName
ctx.Data["description"] = task.Description
ctx.Data["boot_file"] = task.BootFile
- ctx.Data["dataset_name"] = task.DatasetName
+ ctx.Data["dataset_name"] = datasetNames
ctx.Data["work_server_number"] = task.WorkServerNumber
ctx.Data["flavor_name"] = task.FlavorName
ctx.Data["engine_name"] = task.EngineName
@@ -1972,7 +1976,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/"
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
- dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
+ //dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
branchName := form.BranchName
FlavorName := form.FlavorName
EngineName := form.EngineName
@@ -2050,6 +2054,13 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
gitRepo, _ := git.OpenRepository(repo.RepoPath())
commitID, _ := gitRepo.GetBranchCommitID(branchName)
+ _, dataUrl, datasetNames, _, err := getDatasUrlListByUUIDS(uuid)
+ if err != nil {
+ inferenceJobErrorNewDataPrepare(ctx, form)
+ ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsInferenceJobNew, &form)
+ return
+ }
+
if err := downloadCode(repo, codeLocalPath, branchName); err != nil {
log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err)
inferenceJobErrorNewDataPrepare(ctx, form)
@@ -2120,7 +2131,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
req := &modelarts.GenerateInferenceJobReq{
JobName: jobName,
DisplayJobName: displayJobName,
- DataUrl: dataPath,
+ DataUrl: dataUrl,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
@@ -2146,6 +2157,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
ModelVersion: modelVersion,
CkptName: ckptName,
ResultUrl: resultObsPath,
+ DatasetName: datasetNames,
}
err = modelarts.GenerateInferenceJob(ctx, req)
@@ -2700,11 +2712,23 @@ func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string,
datasetInfos := make(map[string]models.DatasetInfo)
attachs, err := models.GetAttachmentsByUUIDs(uuids)
- if err != nil {
+ if err != nil || len(attachs) != len(uuids) {
log.Error("GetAttachmentsByUUIDs failed: %v", err)
return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("GetAttachmentsByUUIDs failed")
}
- for i, attach := range attachs {
+
+ for i, tmpUuid := range uuids {
+ var attach *models.Attachment
+ for _, tmpAttach := range attachs {
+ if tmpAttach.UUID == tmpUuid {
+ attach = tmpAttach
+ break
+ }
+ }
+ if attach == nil {
+ log.Error("GetAttachmentsByUUIDs failed: %v", err)
+ return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("GetAttachmentsByUUIDs failed")
+ }
fileName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(attach.Name, ".zip"), ".tar.gz"), ".tgz")
for _, datasetInfo := range datasetInfos {
if fileName == datasetInfo.Name {
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index a4bc09472..41d34b937 100755
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -1100,7 +1100,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel)
//m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel)
- //m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobNewVersion)
+ //m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir)
//m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
})
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.CloudBrainTrainJobNew)
diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl
index fc2c37bc7..3ca9684a7 100644
--- a/templates/repo/cloudbrain/inference/new.tmpl
+++ b/templates/repo/cloudbrain/inference/new.tmpl
@@ -186,8 +186,10 @@
{{end}}
-
- {{template "custom/select_dataset_train" .}}
+
+ | 数据集文件 | +
|---|
| {{.DatasetName}} | +
| 数据集文件 | +
|---|
| {{.DatasetName}} | +
| 数据集文件 | +
|---|
| {{.DatasetName}} | +
| 数据集文件 | +
|---|
| {{.DatasetName}} | +
| 数据集文件 | +
|---|
| {{.DatasetName}} | +