diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 9c4402868..51186f86b 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1986,7 +1986,6 @@ func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) { return nil, datasetNames, err } for i, attach := range attachs { - //todo: check same name for _, datasetInfo := range datasetInfos { if attach.Name == datasetInfo.Name { log.Error("the dataset name is same: %v", attach.Name) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index a882cefe5..79aeb4cb0 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -51,7 +51,7 @@ const ( Lines = 500 TrainUrl = "train_url" DataUrl = "data_url" - DatasUrl = "datas_url" + MultiDataUrl = "multi_data_url" ResultUrl = "result_url" CkptUrl = "ckpt_url" DeviceTarget = "device_target" diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index bee0eb511..e0f53c6da 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -293,17 +293,17 @@ func NotebookShow(ctx *context.Context) { for _, uuidStr := range uuidList { attachment, err := models.GetAttachmentByUUID(uuidStr) if err == nil { - datasetDownloadLink = datasetDownloadLink + attachment.S3DownloadURL() + ";" + datasetDownloadLink = datasetDownloadLink + attachment.S3DownloadURL() datasetDownload = append(datasetDownload, models.DatasetDownload{ DatasetName: attachment.Name, DatasetDownloadLink: datasetDownloadLink, }) } } - datasetName, err := GetDatasetNameByUUID(task.Uuid) - if err == nil { - task.DatasetName = datasetName - } + // datasetName, err := GetDatasetNameByUUID(task.Uuid) + // if err == nil { + // task.DatasetName = datasetName + // } } } user, err := models.GetUserByID(task.UserID) @@ -986,11 +986,11 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobNew, &form) return } - datasetName, err := GetDatasetNameByUUID(uuid) + _, datasetNames, _, err := models.GetDatasetInfo(uuid) if err != nil { - log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"]) + log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"]) trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr("GetDatasetInfo error", tplModelArtsTrainJobNew, &form) return } @@ -1103,21 +1103,24 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) Value: modelarts.Ascend, }) } - datasetNameHead, dataUrl, datasUrl, isMultiDataset := GetObsDataPathByUUID(datasetName, uuid) + DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid) + if err != nil { + log.Error("Failed to GetDatasUrlListByUUIDS: %v", err) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobNew, &form) + return + } dataPath := dataUrl - var datas models.Datasurl - datas.DatasetUrl = datasUrl - datas.DatasetName = datasetNameHead - jsondatas, err := json.Marshal(datas) + jsondatas, err := json.Marshal(DatasUrlList) if err != nil { log.Error("Failed to Marshal: %v", err) trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("json化参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobNew, &form) return } if isMultiDataset { param = append(param, models.Parameter{ - Label: modelarts.DatasUrl, + Label: modelarts.MultiDataUrl, Value: string(jsondatas), }) } @@ -1187,7 +1190,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) EngineName: EngineName, VersionCount: VersionCount, TotalVersionCount: modelarts.TotalVersionCount, - DatasetName: datasetName, + DatasetName: datasetNames, } //将params转换Parameters.Parameter，出错时返回给前端 @@ -1262,11 +1265,11 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobVersionNew, &form) return } - datasetName, err := GetDatasetNameByUUID(uuid) + _, datasetNames, _, err := models.GetDatasetInfo(uuid) if err != nil { - log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"]) + log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"]) versionErrorDataPrepare(ctx, form) - ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobVersionNew, &form) + ctx.RenderWithErr("GetDatasetInfo error", tplModelArtsTrainJobVersionNew, &form) return } @@ -1354,21 +1357,24 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ Value: modelarts.Ascend, }) } - datasetNameHead, dataUrl, datasUrl, isMultiDataset := GetObsDataPathByUUID(datasetName, uuid) + DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid) + if err != nil { + log.Error("Failed to GetDatasUrlListByUUIDS: %v", err) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) + return + } dataPath := dataUrl - var datas models.Datasurl - datas.DatasetUrl = datasUrl - datas.DatasetName = datasetNameHead - jsondatas, err := json.Marshal(datas) + jsondatas, err := json.Marshal(DatasUrlList) if err != nil { log.Error("Failed to Marshal: %v", err) versionErrorDataPrepare(ctx, form) - ctx.RenderWithErr("json化参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) + ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) return } if isMultiDataset { param = append(param, models.Parameter{ - Label: modelarts.DatasUrl, + Label: modelarts.MultiDataUrl, Value: string(jsondatas), }) } @@ -1445,7 +1451,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ EngineName: EngineName, PreVersionName: PreVersionName, TotalVersionCount: latestTask.TotalVersionCount + 1, - DatasetName: datasetName, + DatasetName: datasetNames, } err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) @@ -2491,61 +2497,37 @@ func TrainJobDownloadLogFile(ctx *context.Context) { ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } -func GetObsDataPathByUUID(datasetName string, uuidStr string) (string, string, string, bool) { +func GetDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, bool, error) { var dataUrl string - var datasUrl string - var datasetNameHead string + var DatasUrlList []models.Datasurl uuidList := strings.Split(uuidStr, ";") - datasetNameList := strings.Split(datasetName, ";") - for k, datasetName := range datasetNameList { - index := strings.LastIndex(datasetName, ".") - if index <= 0 { - index = 0 - } - if k == 0 { - datasetHead := datasetName[:index] - datasetNameHead = datasetNameHead + datasetHead - } else if k > 0 && datasetName != "" { - datasetHead := ";" + datasetName[:index] - datasetNameHead = datasetNameHead + datasetHead - } - - } + isMultiDataset := false - if len(uuidList) <= 1 { - dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuidStr[0:1], uuidStr[1:2]) + "/" + uuidStr + uuidStr + "/" - datasUrl = "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuidStr[0:1], uuidStr[1:2]) + "/" + uuidStr + uuidStr + "/" - isMultiDataset := false - return datasetNameHead, dataUrl, datasUrl, isMultiDataset - } else { - for k, uuid := range uuidList { - if k == 0 { - datasUrlNext := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - datasUrl = datasUrl + datasUrlNext + if len(uuidList) > 1 { + for _, uuid := range uuidList { + attach, err := models.GetAttachmentByUUID(uuid) + if err != nil { + log.Error("GetAttachmentByUUID failed: %v", err) + return nil, "", isMultiDataset, err } - if k > 0 { - datasUrlNext := ";" + "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" - datasUrl = datasUrl + datasUrlNext + datasetName := attach.Name + index := strings.LastIndex(datasetName, ".") + if index <= 0 { + index = 0 } + datasetNameHead := datasetName[:index] + datasetUrl := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + DatasUrlList = append(DatasUrlList, models.Datasurl{ + DatasetUrl: datasetUrl, + DatasetName: datasetNameHead, + }) } firstDataset := uuidList[0] dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(firstDataset[0:1], firstDataset[1:2]) + "/" + firstDataset + firstDataset + "/" - isMultiDataset := true - return datasetNameHead, dataUrl, datasUrl, isMultiDataset - } -} -func GetDatasetNameByUUID(uuid string) (string, error) { - uuidList := strings.Split(uuid, ";") - var datasetName string - for _, uuidStr := range uuidList { - attach, err := models.GetAttachmentByUUID(uuidStr) - if err != nil { - log.Error("GetAttachmentByUUID failed:%v", err) - return "", err - } - datasetName = datasetName + attach.Name + ";" + isMultiDataset = true + return DatasUrlList, dataUrl, isMultiDataset, nil } - return datasetName, nil + return nil, "", isMultiDataset, nil } func IsDatasetUseCountExceed(uuid string) bool { uuidList := strings.Split(uuid, ";")