Merge branch 'multi-dataset' of https://git.openi.org.cn/OpenI/aiforge into multi-dataset

models/cloudbrain.go

@@ -1986,7 +1986,6 @@ func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) {
 		return nil, datasetNames, err
 	}
 	for i, attach := range attachs {
-		//todo: check same name
 		for _, datasetInfo := range datasetInfos {
 			if attach.Name == datasetInfo.Name {
 				log.Error("the dataset name is same: %v", attach.Name)

modules/modelarts/modelarts.go

@@ -51,7 +51,7 @@ const (
 	Lines            = 500
 	TrainUrl         = "train_url"
 	DataUrl          = "data_url"
-	DatasUrl         = "datas_url"
+	MultiDataUrl     = "multi_data_url"
 	ResultUrl        = "result_url"
 	CkptUrl          = "ckpt_url"
 	DeviceTarget     = "device_target"

routers/repo/modelarts.go

@@ -293,17 +293,17 @@ func NotebookShow(ctx *context.Context) {
 			for _, uuidStr := range uuidList {
 				attachment, err := models.GetAttachmentByUUID(uuidStr)
 				if err == nil {
-					datasetDownloadLink = datasetDownloadLink + attachment.S3DownloadURL() + ";"
+					datasetDownloadLink = datasetDownloadLink + attachment.S3DownloadURL()
 					datasetDownload = append(datasetDownload, models.DatasetDownload{
 						DatasetName:         attachment.Name,
 						DatasetDownloadLink: datasetDownloadLink,
 					})
 				}
 			}
-			datasetName, err := GetDatasetNameByUUID(task.Uuid)
-			if err == nil {
-				task.DatasetName = datasetName
-			}
+			// datasetName, err := GetDatasetNameByUUID(task.Uuid)
+			// if err == nil {
+			// 	task.DatasetName = datasetName
+			// }
 		}
 	}
 	user, err := models.GetUserByID(task.UserID)
@@ -986,11 +986,11 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobNew, &form)
 		return
 	}
-	datasetName, err := GetDatasetNameByUUID(uuid)
+	_, datasetNames, _, err := models.GetDatasetInfo(uuid)
 	if err != nil {
-		log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"])
+		log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"])
 		trainJobErrorNewDataPrepare(ctx, form)
-		ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobNew, &form)
+		ctx.RenderWithErr("GetDatasetInfo error", tplModelArtsTrainJobNew, &form)
 		return
 	}
 
@@ -1103,21 +1103,24 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 			Value: modelarts.Ascend,
 		})
 	}
-	datasetNameHead, dataUrl, datasUrl, isMultiDataset := GetObsDataPathByUUID(datasetName, uuid)
+	DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid)
+	if err != nil {
+		log.Error("Failed to GetDatasUrlListByUUIDS: %v", err)
+		trainJobErrorNewDataPrepare(ctx, form)
+		ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobNew, &form)
+		return
+	}
 	dataPath := dataUrl
-	var datas models.Datasurl
-	datas.DatasetUrl = datasUrl
-	datas.DatasetName = datasetNameHead
-	jsondatas, err := json.Marshal(datas)
+	jsondatas, err := json.Marshal(DatasUrlList)
 	if err != nil {
 		log.Error("Failed to Marshal: %v", err)
 		trainJobErrorNewDataPrepare(ctx, form)
-		ctx.RenderWithErr("json化参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form)
+		ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobNew, &form)
 		return
 	}
 	if isMultiDataset {
 		param = append(param, models.Parameter{
-			Label: modelarts.DatasUrl,
+			Label: modelarts.MultiDataUrl,
 			Value: string(jsondatas),
 		})
 	}
@@ -1187,7 +1190,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		EngineName:        EngineName,
 		VersionCount:      VersionCount,
 		TotalVersionCount: modelarts.TotalVersionCount,
-		DatasetName:       datasetName,
+		DatasetName:       datasetNames,
 	}
 
 	//将params转换Parameters.Parameter，出错时返回给前端
@@ -1262,11 +1265,11 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		ctx.RenderWithErr("DatasetUseCount is Exceed", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
-	datasetName, err := GetDatasetNameByUUID(uuid)
+	_, datasetNames, _, err := models.GetDatasetInfo(uuid)
 	if err != nil {
-		log.Error("GetDatasetNameByUUID failed:%v", err, ctx.Data["MsgID"])
+		log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"])
 		versionErrorDataPrepare(ctx, form)
-		ctx.RenderWithErr("GetDatasetNameByUUID error", tplModelArtsTrainJobVersionNew, &form)
+		ctx.RenderWithErr("GetDatasetInfo error", tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 
@@ -1354,21 +1357,24 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 			Value: modelarts.Ascend,
 		})
 	}
-	datasetNameHead, dataUrl, datasUrl, isMultiDataset := GetObsDataPathByUUID(datasetName, uuid)
+	DatasUrlList, dataUrl, isMultiDataset, err := GetDatasUrlListByUUIDS(uuid)
+	if err != nil {
+		log.Error("Failed to GetDatasUrlListByUUIDS: %v", err)
+		versionErrorDataPrepare(ctx, form)
+		ctx.RenderWithErr("Failed to GetDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
+		return
+	}
 	dataPath := dataUrl
-	var datas models.Datasurl
-	datas.DatasetUrl = datasUrl
-	datas.DatasetName = datasetNameHead
-	jsondatas, err := json.Marshal(datas)
+	jsondatas, err := json.Marshal(DatasUrlList)
 	if err != nil {
 		log.Error("Failed to Marshal: %v", err)
 		versionErrorDataPrepare(ctx, form)
-		ctx.RenderWithErr("json化参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
+		ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
 		return
 	}
 	if isMultiDataset {
 		param = append(param, models.Parameter{
-			Label: modelarts.DatasUrl,
+			Label: modelarts.MultiDataUrl,
 			Value: string(jsondatas),
 		})
 	}
@@ -1445,7 +1451,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		EngineName:        EngineName,
 		PreVersionName:    PreVersionName,
 		TotalVersionCount: latestTask.TotalVersionCount + 1,
-		DatasetName:       datasetName,
+		DatasetName:       datasetNames,
 	}
 
 	err = modelarts.GenerateTrainJobVersion(ctx, req, jobID)
@@ -2491,61 +2497,37 @@ func TrainJobDownloadLogFile(ctx *context.Context) {
 	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
-func GetObsDataPathByUUID(datasetName string, uuidStr string) (string, string, string, bool) {
+func GetDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, bool, error) {
 	var dataUrl string
-	var datasUrl string
-	var datasetNameHead string
+	var DatasUrlList []models.Datasurl
 	uuidList := strings.Split(uuidStr, ";")
-	datasetNameList := strings.Split(datasetName, ";")
-	for k, datasetName := range datasetNameList {
-		index := strings.LastIndex(datasetName, ".")
-		if index <= 0 {
-			index = 0
-		}
-		if k == 0 {
-			datasetHead := datasetName[:index]
-			datasetNameHead = datasetNameHead + datasetHead
-		} else if k > 0 && datasetName != "" {
-			datasetHead := ";" + datasetName[:index]
-			datasetNameHead = datasetNameHead + datasetHead
-		}
-
-	}
+	isMultiDataset := false
 
-	if len(uuidList) <= 1 {
-		dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuidStr[0:1], uuidStr[1:2]) + "/" + uuidStr + uuidStr + "/"
-		datasUrl = "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuidStr[0:1], uuidStr[1:2]) + "/" + uuidStr + uuidStr + "/"
-		isMultiDataset := false
-		return datasetNameHead, dataUrl, datasUrl, isMultiDataset
-	} else {
-		for k, uuid := range uuidList {
-			if k == 0 {
-				datasUrlNext := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-				datasUrl = datasUrl + datasUrlNext
+	if len(uuidList) > 1 {
+		for _, uuid := range uuidList {
+			attach, err := models.GetAttachmentByUUID(uuid)
+			if err != nil {
+				log.Error("GetAttachmentByUUID failed: %v", err)
+				return nil, "", isMultiDataset, err
 			}
-			if k > 0 {
-				datasUrlNext := ";" + "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-				datasUrl = datasUrl + datasUrlNext
+			datasetName := attach.Name
+			index := strings.LastIndex(datasetName, ".")
+			if index <= 0 {
+				index = 0
 			}
+			datasetNameHead := datasetName[:index]
+			datasetUrl := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
+			DatasUrlList = append(DatasUrlList, models.Datasurl{
+				DatasetUrl:  datasetUrl,
+				DatasetName: datasetNameHead,
+			})
 		}
 		firstDataset := uuidList[0]
 		dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(firstDataset[0:1], firstDataset[1:2]) + "/" + firstDataset + firstDataset + "/"
-		isMultiDataset := true
-		return datasetNameHead, dataUrl, datasUrl, isMultiDataset
-	}
-}
-func GetDatasetNameByUUID(uuid string) (string, error) {
-	uuidList := strings.Split(uuid, ";")
-	var datasetName string
-	for _, uuidStr := range uuidList {
-		attach, err := models.GetAttachmentByUUID(uuidStr)
-		if err != nil {
-			log.Error("GetAttachmentByUUID failed:%v", err)
-			return "", err
-		}
-		datasetName = datasetName + attach.Name + ";"
+		isMultiDataset = true
+		return DatasUrlList, dataUrl, isMultiDataset, nil
 	}
-	return datasetName, nil
+	return nil, "", isMultiDataset, nil
 }
 func IsDatasetUseCountExceed(uuid string) bool {
 	uuidList := strings.Split(uuid, ";")