// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package storage import ( "errors" "io" "net/url" "path" "sort" "strconv" "strings" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" "github.com/unknwon/com" ) type FileInfo struct { FileName string `json:"FileName"` ModTime string `json:"ModTime"` IsDir bool `json:"IsDir"` Size int64 `json:"Size"` ParenDir string `json:"ParenDir"` UUID string `json:"UUID"` } type FileInfoList []FileInfo func (ulist FileInfoList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } func (ulist FileInfoList) Len() int { return len(ulist) } func (ulist FileInfoList) Less(i, j int) bool { return strings.Compare(ulist[i].FileName, ulist[j].FileName) > 0 } //check if has the object func ObsHasObject(path string) (bool, error) { hasObject := false input := &obs.GetObjectMetadataInput{} input.Bucket = setting.Bucket input.Key = path _, err := ObsCli.GetObjectMetadata(input) if err == nil { hasObject = true } else { if obsError, ok := err.(obs.ObsError); ok { log.Error("GetObjectMetadata failed(%d): %s", obsError.StatusCode, obsError.Message) } else { log.Error("%v", err.Error()) } } return hasObject, nil } func GetObsPartInfos(uuid, uploadID, fileName string) (string, error) { key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") output, err := ObsCli.ListParts(&obs.ListPartsInput{ Bucket: setting.Bucket, Key: key, UploadId: uploadID, }) if err != nil { log.Error("ListParts failed:", err.Error()) return "", err } var chunks string for _, partInfo := range output.Parts { chunks += strconv.Itoa(partInfo.PartNumber) + "-" + partInfo.ETag + "," } return chunks, nil } func NewObsMultiPartUpload(uuid, fileName string) (string, error) { input := &obs.InitiateMultipartUploadInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") output, err := ObsCli.InitiateMultipartUpload(input) if err != nil { log.Error("InitiateMultipartUpload failed:", err.Error()) return "", err } return output.UploadId, nil } func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { input := &obs.CompleteMultipartUploadInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadID output, err := ObsCli.ListParts(&obs.ListPartsInput{ Bucket: setting.Bucket, Key: input.Key, UploadId: uploadID, }) if err != nil { log.Error("ListParts failed:", err.Error()) return err } for _, partInfo := range output.Parts { input.Parts = append(input.Parts, obs.Part{ PartNumber: partInfo.PartNumber, ETag: partInfo.ETag, }) } _, err = ObsCli.CompleteMultipartUpload(input) if err != nil { log.Error("CompleteMultipartUpload failed:", err.Error()) return err } return nil } func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { input := &obs.UploadPartInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadId input.PartNumber = partNumber input.Body = putBody output, err := ObsCli.UploadPart(input) if err == nil { log.Info("RequestId:%s\n", output.RequestId) log.Info("ETag:%s\n", output.ETag) return nil } else { if obsError, ok := err.(obs.ObsError); ok { log.Info(obsError.Code) log.Info(obsError.Message) return obsError } else { log.Error("error:", err.Error()) return err } } } //delete all file under the dir path func ObsRemoveObject(bucket string, path string) error { log.Info("Bucket=" + bucket + " path=" + path) if len(path) == 0 { return errors.New("path canot be null.") } input := &obs.ListObjectsInput{} input.Bucket = bucket // 设置每页100个对象 input.MaxKeys = 100 input.Prefix = path index := 1 log.Info("prefix=" + input.Prefix) for { output, err := ObsCli.ListObjects(input) if err == nil { log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { log.Info("delete obs file:" + val.Key) delObj := &obs.DeleteObjectInput{} delObj.Bucket = setting.Bucket delObj.Key = val.Key ObsCli.DeleteObject(delObj) } if output.IsTruncated { input.Marker = output.NextMarker } else { break } } else { if obsError, ok := err.(obs.ObsError); ok { log.Info("Code:%s\n", obsError.Code) log.Info("Message:%s\n", obsError.Message) } return err } } return nil } func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = bucket input.Key = key output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { return ObsDownloadAFile(setting.Bucket, strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")) } func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, JobName, setting.OutPutPath, fileName), "/") // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPath string) (int64, error) { input := &obs.ListObjectsInput{} input.Bucket = srcBucket // 设置每页100个对象 input.MaxKeys = 100 input.Prefix = srcPath index := 1 length := len(srcPath) var fileTotalSize int64 log.Info("prefix=" + input.Prefix) for { output, err := ObsCli.ListObjects(input) if err == nil { log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { destKey := destPath + val.Key[length:] obsCopyFile(srcBucket, val.Key, destBucket, destKey) fileTotalSize += val.Size } if output.IsTruncated { input.Marker = output.NextMarker } else { break } } else { if obsError, ok := err.(obs.ObsError); ok { log.Info("Code:%s\n", obsError.Code) log.Info("Message:%s\n", obsError.Message) } return 0, err } } return fileTotalSize, nil } func obsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { input := &obs.CopyObjectInput{} input.Bucket = destBucket input.Key = destKeyName input.CopySourceBucket = srcBucket input.CopySourceKey = srcKeyName _, err := ObsCli.CopyObject(input) if err == nil { log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) } else { log.Info("copy failed,,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) if obsError, ok := err.(obs.ObsError); ok { log.Info(obsError.Code) log.Info(obsError.Message) } return err } return nil } func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefixRootPath + relativePath if !strings.HasSuffix(input.Prefix, "/") { input.Prefix += "/" } output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) prefixLen := len(input.Prefix) if err == nil { for _, val := range output.Contents { log.Info("val key=" + val.Key) var isDir bool var fileName string if val.Key == input.Prefix { continue } if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { continue } if strings.HasSuffix(val.Key, "/") { isDir = true fileName = val.Key[prefixLen : len(val.Key)-1] relativePath += val.Key[prefixLen:] } else { isDir = false fileName = val.Key[prefixLen:] } fileInfo := FileInfo{ ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), FileName: fileName, Size: val.Size, IsDir: isDir, ParenDir: relativePath, } fileInfos = append(fileInfos, fileInfo) } return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } return nil, err } } func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket // 设置每页100个对象 input.MaxKeys = 100 input.Prefix = prefix index := 1 fileInfoList := FileInfoList{} prefixLen := len(prefix) log.Info("prefix=" + input.Prefix) for { output, err := ObsCli.ListObjects(input) if err == nil { log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { var isDir bool if prefixLen == len(val.Key) { continue } if strings.HasSuffix(val.Key, "/") { isDir = true } else { isDir = false } fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: val.Key[prefixLen:], Size: val.Size, IsDir: isDir, ParenDir: "", } fileInfoList = append(fileInfoList, fileInfo) } if output.IsTruncated { input.Marker = output.NextMarker } else { break } } else { if obsError, ok := err.(obs.ObsError); ok { log.Info("Code:%s\n", obsError.Code) log.Info("Message:%s\n", obsError.Message) } return nil, err } } sort.Sort(fileInfoList) return fileInfoList, nil } func GetObsListObject(jobName, outPutPath, parentDir, versionName string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, outPutPath, versionName, parentDir), "/") strPrefix := strings.Split(input.Prefix, "/") output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { //dirs in next level dir if len(str1)-len(strPrefix) > 2 { continue } fileName = str1[len(str1)-2] isDir = true if parentDir == "" { nextParentDir = fileName } else { nextParentDir = parentDir + "/" + fileName } if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == outPutPath { continue } } else { //files in next level dir if len(str1)-len(strPrefix) > 1 { continue } fileName = str1[len(str1)-1] isDir = false nextParentDir = parentDir } fileInfo := FileInfo{ ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), FileName: fileName, Size: val.Size, IsDir: isDir, ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) } sort.Slice(fileInfos, func(i, j int) bool { return fileInfos[i].ModTime > fileInfos[j].ModTime }) return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } return nil, err } } func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Expires = 60 * 60 input.Method = obs.HttpMethodPut input.QueryParams = map[string]string{ "partNumber": com.ToStr(partNumber, 10), "uploadId": uploadId, //"partSize": com.ToStr(partSize,10), } output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = bucket input.Key = key input.Expires = 60 * 60 input.Method = obs.HttpMethodGet comma := strings.LastIndex(key, "/") filename := key if comma != -1 { filename = key[comma+1:] } reqParams := make(map[string]string) filename = url.PathEscape(filename) reqParams["response-content-disposition"] = "attachment; filename=\"" + filename + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")) } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Method = obs.HttpMethodGet input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Bucket = setting.Bucket input.Expires = 60 * 60 fileName = url.PathEscape(fileName) reqParams := make(map[string]string) reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" input.QueryParams = reqParams output, err := ObsCli.CreateSignedUrl(input) if err != nil { log.Error("CreateSignedUrl failed:", err.Error()) return "", err } return output.SignedUrl, nil } func ObsCreateObject(path string) error { input := &obs.PutObjectInput{} input.Bucket = setting.Bucket input.Key = path _, err := ObsCli.PutObject(input) if err != nil { log.Error("PutObject failed:", err.Error()) return err } return nil }