// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package storage import ( "io" "path" "strconv" "strings" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" "github.com/unknwon/com" ) type FileInfo struct { FileName string `json:"FileName"` ModTime string `json:"ModTime"` IsDir bool `json:"IsDir"` Size int64 `json:"Size"` ParenDir string `json:"ParenDir"` UUID string `json:"UUID"` } //check if has the object //todo:修改查询方式 func ObsHasObject(path string) (bool, error) { hasObject := false output, err := ObsCli.ListObjects(&obs.ListObjectsInput{Bucket: setting.Bucket}) if err != nil { log.Error("ListObjects failed:%v", err) return hasObject, err } for _, obj := range output.Contents { //obj.Key:attachment/0/1/019fd24e-4ef7-41cc-9f85-4a7b8504d958 if path == obj.Key { hasObject = true break } } return hasObject, nil } func GetObsPartInfos(uuid string, uploadID string) (string, error) { key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/") output, err := ObsCli.ListParts(&obs.ListPartsInput{ Bucket: setting.Bucket, Key: key, UploadId: uploadID, }) if err != nil { log.Error("ListParts failed:", err.Error()) return "", err } var chunks string for _, partInfo := range output.Parts { chunks += strconv.Itoa(partInfo.PartNumber) + "-" + partInfo.ETag + "," } return chunks, nil } func NewObsMultiPartUpload(uuid, fileName string) (string, error) { input := &obs.InitiateMultipartUploadInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") output, err := ObsCli.InitiateMultipartUpload(input) if err != nil { log.Error("InitiateMultipartUpload failed:", err.Error()) return "", err } return output.UploadId, nil } func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { input := &obs.CompleteMultipartUploadInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadID output, err := ObsCli.ListParts(&obs.ListPartsInput{ Bucket: setting.Bucket, Key: input.Key, UploadId: uploadID, }) if err != nil { log.Error("ListParts failed:", err.Error()) return err } for _, partInfo := range output.Parts { input.Parts = append(input.Parts, obs.Part{ PartNumber: partInfo.PartNumber, ETag: partInfo.ETag, }) } _, err = ObsCli.CompleteMultipartUpload(input) if err != nil { log.Error("CompleteMultipartUpload failed:", err.Error()) return err } return nil } func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { input := &obs.UploadPartInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadId input.PartNumber = partNumber input.Body = putBody output, err := ObsCli.UploadPart(input) if err == nil { log.Info("RequestId:%s\n", output.RequestId) log.Info("ETag:%s\n", output.ETag) return nil } else { if obsError, ok := err.(obs.ObsError); ok { log.Info(obsError.Code) log.Info(obsError.Message) return obsError } else { log.Error("error:", err.Error()) return err } } } func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, JobName, setting.OutPutPath, fileName), "/") // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") strPrefix := strings.Split(input.Prefix, "/") output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { //dirs in next level dir if len(str1)-len(strPrefix) > 2 { continue } fileName = str1[len(str1)-2] isDir = true if parentDir == "" { nextParentDir = fileName } else { nextParentDir = parentDir + "/" + fileName } if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { continue } } else { //files in next level dir if len(str1)-len(strPrefix) > 1 { continue } fileName = str1[len(str1)-1] isDir = false nextParentDir = parentDir } fileInfo := FileInfo{ ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), FileName: fileName, Size: val.Size, IsDir: isDir, ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) } return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } return nil, err } } func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Expires = 60 * 60 input.Method = obs.HttpMethodPut input.QueryParams = map[string]string{ "partNumber": com.ToStr(partNumber, 10), "uploadId": uploadId, //"partSize": com.ToStr(partSize,10), } output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") input.Expires = 60 * 60 input.Method = obs.HttpMethodGet reqParams := make(map[string]string) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Method = obs.HttpMethodGet input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Bucket = setting.Bucket input.Expires = 60 * 60 reqParams := make(map[string]string) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func ObsCreateObject(path string) error { input := &obs.PutObjectInput{} input.Bucket = setting.Bucket input.Key = path _, err := ObsCli.PutObject(input) if err != nil { log.Error("PutObject failed:", err.Error()) return err } return nil }