diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9ad232437..e4dd5ec2f 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1080,3 +1080,8 @@ BASE_PATH = attachment/ [modelarts] ENDPOINT = 112.95.163.80 +PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa +PROJECT_NAME = cn-south-222_test +USERNAME = test1 +PASSWORD = Qizhi@test. +DOMAIN = cn-south-222 diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 182a13a61..80fceb394 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -14,6 +14,7 @@ import ( type CloudbrainStatus string type JobType string +type ModelArtsJobStatus string const ( JobWaiting CloudbrainStatus = "WAITING" @@ -24,6 +25,22 @@ const ( JobTypeDebug JobType = "DEBUG" JobTypeBenchmark JobType = "BENCHMARK" + + ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 + ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 + ModelArtsCreateFailed ModelArtsJobStatus = "CREATE_FAILED" //创建失败 + ModelArtsStartQueuing ModelArtsJobStatus = "START_QUEUING" //免费资源启动排队中 + ModelArtsReadyToStart ModelArtsJobStatus = "READY_TO_START" //免费资源等待启动 + ModelArtsStarting ModelArtsJobStatus = "STARTING" //启动中 + ModelArtsRestarting ModelArtsJobStatus = "RESTARTING" //重启中 + ModelArtsStartFailed ModelArtsJobStatus = "START_FAILED" //启动失败 + ModelArtsRunning ModelArtsJobStatus = "RUNNING" //运行中 + ModelArtsStopping ModelArtsJobStatus = "STOPPING" //停止中 + ModelArtsStopped ModelArtsJobStatus = "STOPPED" //停止 + ModelArtsUnavailable ModelArtsJobStatus = "UNAVAILABLE" //故障 + ModelArtsDeleted ModelArtsJobStatus = "DELETED" //已删除 + ModelArtsResizing ModelArtsJobStatus = "RESIZING" //规格变更中 + ModelArtsResizFailed ModelArtsJobStatus = "RESIZE_FAILED" //规格变更失败 ) type Cloudbrain struct { @@ -41,6 +58,7 @@ type Cloudbrain struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` + Type int `xorm:"INDEX DEFAULT 0"` User *User `xorm:"-"` Repo *Repository `xorm:"-"` @@ -112,6 +130,7 @@ type CloudbrainsOptions struct { SortType string CloudbrainIDs []int64 // JobStatus CloudbrainStatus + Type int } type TaskPod struct { TaskRoleStatus struct { @@ -257,6 +276,137 @@ type StopJobResult struct { Msg string `json:"msg"` } +type CreateNotebookParams struct { + JobName string `json:"name"` + Description string `json:"description"` + ProfileID string `json:"profile_id"` + Flavor string `json:"flavor"` + Spec Spec `json:"spec"` + Workspace Workspace `json:"workspace"` +} + +type Workspace struct { + ID string `json:"id"` +} + +type Spec struct { + Storage Storage `json:"storage"` + AutoStop AutoStop `json:"auto_stop"` +} + +type AutoStop struct { + Enable bool `json:"enable"` + Duration int `json:"duration"` +} + +type Storage struct { + Type string `json:"type"` + Location Location `json:"location"` +} + +type Location struct { + Path string `json:"path"` +} + +type CreateNotebookResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Status string `json:"status"` + CreationTimestamp string `json:"creation_timestamp"` + LatestUpdateTimestamp string `json:"latest_update_timestamp"` + Profile struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DeType string `json:"de_type"` + FlavorType string `json:"flavor_type"` + } `json:"profile"` + Flavor string `json:"flavor"` + FlavorDetails struct{ + Name string `json:"name"` + Status string `json:"status"` + QueuingNum int `json:"queuing_num"` + QueueLeftTime int `json:"queue_left_time"` //s + Duration int `json:"duration"` //auto_stop_time s + } `json:"flavor_details"` +} + +type GetNotebookResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Status string `json:"status"` + CreationTimestamp string `json:"creation_timestamp"` + LatestUpdateTimestamp string `json:"latest_update_timestamp"` + Profile struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DeType string `json:"de_type"` + FlavorType string `json:"flavor_type"` + } `json:"profile"` + Flavor string `json:"flavor"` + FlavorDetails struct{ + Name string `json:"name"` + Status string `json:"status"` + QueuingNum int `json:"queuing_num"` + QueueLeftTime int `json:"queue_left_time"` //s + Duration int `json:"duration"` //auto_stop_time s + } `json:"flavor_details"` + QueuingInfo struct{ + ID string `json:"id"` + Name string `json:"name"` + Flavor string `json:"flavor"` + DeType string `json:"de_type"` + Status string `json:"status"` + BeginTimestamp int `json:"begin_timestamp"`//time of instance begin in queue + RemainTime int `json:"remain_time"` //remain time of instance + EndTimestamp int `json:"end_timestamp"` // + Rank int `json:"rank"` //rank of instance in queue + } `json:"queuing_info"` +} + +type GetTokenParams struct { + Auth Auth `json:auth` +} + +type Auth struct { + Identity Identity `json:identity` + Scope Scope `json:scope` +} + +type Scope struct { + Project Project `json:project` +} + +type Project struct { + Name string `json:name` +} + +type Identity struct { + Methods []string `json:"methods"` + Password Password `json:password` +} + +type Password struct { + User NotebookUser `json:user` +} + +type NotebookUser struct { + Name string `json:name` + Password string `json:"password"` + Domain Domain `json:domain` +} + +type Domain struct { + Name string `json:name` +} + func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { sess := x.NewSession() defer sess.Close() @@ -280,6 +430,12 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*Cloudbrain, int64, error) { ) } + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"cloudbrain.type": opts.Type}, + ) + } + // switch opts.JobStatus { // case JobWaiting: // cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) diff --git a/models/user.go b/models/user.go index 4d282f477..2a150187e 100755 --- a/models/user.go +++ b/models/user.go @@ -2045,8 +2045,8 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) - err := x.Where("public_key is null"). - Or("private_key is null"). + err := x.Where("public_key = ''"). + Or("private_key = ''"). Find(&users) return users, err } diff --git a/modules/APIGW-go-sdk-2.0.2/core/escape.go b/modules/APIGW-go-sdk-2.0.2/core/escape.go new file mode 100755 index 000000000..e8c76b8ae --- /dev/null +++ b/modules/APIGW-go-sdk-2.0.2/core/escape.go @@ -0,0 +1,42 @@ +// based on https://github.com/golang/go/blob/master/src/net/url/url.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package core + +func shouldEscape(c byte) bool { + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c == '-' || c == '~' || c == '.' { + return false + } + return true +} +func escape(s string) string { + hexCount := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + hexCount++ + } + } + + if hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case shouldEscape(c): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} diff --git a/modules/APIGW-go-sdk-2.0.2/core/signer.go b/modules/APIGW-go-sdk-2.0.2/core/signer.go new file mode 100755 index 000000000..7992713b3 --- /dev/null +++ b/modules/APIGW-go-sdk-2.0.2/core/signer.go @@ -0,0 +1,208 @@ +// HWS API Gateway Signature +// based on https://github.com/datastream/aws/blob/master/signv4.go +// Copyright (c) 2014, Xianjie + +package core + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "fmt" + "io/ioutil" + "net/http" + "sort" + "strings" + "time" +) + +const ( + BasicDateFormat = "20060102T150405Z" + Algorithm = "SDK-HMAC-SHA256" + HeaderXDate = "X-Sdk-Date" + HeaderHost = "host" + HeaderAuthorization = "Authorization" + HeaderContentSha256 = "X-Sdk-Content-Sha256" +) + +func hmacsha256(key []byte, data string) ([]byte, error) { + h := hmac.New(sha256.New, []byte(key)) + if _, err := h.Write([]byte(data)); err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +// Build a CanonicalRequest from a regular request string +// +// CanonicalRequest = +// HTTPRequestMethod + '\n' + +// CanonicalURI + '\n' + +// CanonicalQueryString + '\n' + +// CanonicalHeaders + '\n' + +// SignedHeaders + '\n' + +// HexEncode(Hash(RequestPayload)) +func CanonicalRequest(r *http.Request, signedHeaders []string) (string, error) { + var hexencode string + var err error + if hex := r.Header.Get(HeaderContentSha256); hex != "" { + hexencode = hex + } else { + data, err := RequestPayload(r) + if err != nil { + return "", err + } + hexencode, err = HexEncodeSHA256Hash(data) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, CanonicalURI(r), CanonicalQueryString(r), CanonicalHeaders(r, signedHeaders), strings.Join(signedHeaders, ";"), hexencode), err +} + +// CanonicalURI returns request uri +func CanonicalURI(r *http.Request) string { + pattens := strings.Split(r.URL.Path, "/") + var uri []string + for _, v := range pattens { + uri = append(uri, escape(v)) + } + urlpath := strings.Join(uri, "/") + if len(urlpath) == 0 || urlpath[len(urlpath)-1] != '/' { + urlpath = urlpath + "/" + } + return urlpath +} + +// CanonicalQueryString +func CanonicalQueryString(r *http.Request) string { + var keys []string + query := r.URL.Query() + for key := range query { + keys = append(keys, key) + } + sort.Strings(keys) + var a []string + for _, key := range keys { + k := escape(key) + sort.Strings(query[key]) + for _, v := range query[key] { + kv := fmt.Sprintf("%s=%s", k, escape(v)) + a = append(a, kv) + } + } + queryStr := strings.Join(a, "&") + r.URL.RawQuery = queryStr + return queryStr +} + +// CanonicalHeaders +func CanonicalHeaders(r *http.Request, signerHeaders []string) string { + var a []string + header := make(map[string][]string) + for k, v := range r.Header { + header[strings.ToLower(k)] = v + } + for _, key := range signerHeaders { + value := header[key] + if strings.EqualFold(key, HeaderHost) { + value = []string{r.Host} + } + sort.Strings(value) + for _, v := range value { + a = append(a, key+":"+strings.TrimSpace(v)) + } + } + return fmt.Sprintf("%s\n", strings.Join(a, "\n")) +} + +// SignedHeaders +func SignedHeaders(r *http.Request) []string { + var a []string + for key := range r.Header { + a = append(a, strings.ToLower(key)) + } + sort.Strings(a) + return a +} + +// RequestPayload +func RequestPayload(r *http.Request) ([]byte, error) { + if r.Body == nil { + return []byte(""), nil + } + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return []byte(""), err + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + return b, err +} + +// Create a "String to Sign". +func StringToSign(canonicalRequest string, t time.Time) (string, error) { + hash := sha256.New() + _, err := hash.Write([]byte(canonicalRequest)) + if err != nil { + return "", err + } + return fmt.Sprintf("%s\n%s\n%x", + Algorithm, t.UTC().Format(BasicDateFormat), hash.Sum(nil)), nil +} + +// Create the HWS Signature. +func SignStringToSign(stringToSign string, signingKey []byte) (string, error) { + hm, err := hmacsha256(signingKey, stringToSign) + return fmt.Sprintf("%x", hm), err +} + +// HexEncodeSHA256Hash returns hexcode of sha256 +func HexEncodeSHA256Hash(body []byte) (string, error) { + hash := sha256.New() + if body == nil { + body = []byte("") + } + _, err := hash.Write(body) + return fmt.Sprintf("%x", hash.Sum(nil)), err +} + +// Get the finalized value for the "Authorization" header. The signature parameter is the output from SignStringToSign +func AuthHeaderValue(signature, accessKey string, signedHeaders []string) string { + return fmt.Sprintf("%s Access=%s, SignedHeaders=%s, Signature=%s", Algorithm, accessKey, strings.Join(signedHeaders, ";"), signature) +} + +// Signature HWS meta +type Signer struct { + Key string + Secret string +} + +// SignRequest set Authorization header +func (s *Signer) Sign(r *http.Request) error { + var t time.Time + var err error + var dt string + if dt = r.Header.Get(HeaderXDate); dt != "" { + t, err = time.Parse(BasicDateFormat, dt) + } + if err != nil || dt == "" { + t = time.Now() + r.Header.Set(HeaderXDate, t.UTC().Format(BasicDateFormat)) + } + signedHeaders := SignedHeaders(r) + canonicalRequest, err := CanonicalRequest(r, signedHeaders) + if err != nil { + return err + } + stringToSign, err := StringToSign(canonicalRequest, t) + if err != nil { + return err + } + signature, err := SignStringToSign(stringToSign, []byte(s.Secret)) + if err != nil { + return err + } + authValue := AuthHeaderValue(signature, s.Key, signedHeaders) + r.Header.Set(HeaderAuthorization, authValue) + return nil +} diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index 2b150cf75..2470c2ad6 100755 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -5,7 +5,6 @@ import ( "gitea.com/macaron/macaron" ) -// CreateDatasetForm form for dataset page type CreateCloudBrainForm struct { JobName string `form:"job_name" binding:"Required"` Image string `form:"image" binding:"Required"` diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go new file mode 100755 index 000000000..b15de3c37 --- /dev/null +++ b/modules/auth/modelarts.go @@ -0,0 +1,18 @@ +package auth + +import ( + "gitea.com/macaron/binding" + "gitea.com/macaron/macaron" +) + +type CreateModelArtsForm struct { + JobName string `form:"job_name" binding:"Required"` + Image string `form:"image" binding:"Required"` + Command string `form:"command" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + JobType string `form:"job_type" binding:"Required"` +} + +func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index dedfb55f6..50de926ca 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -98,6 +98,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, + Type: models.TypeCloudBrainOne, }) if err != nil { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go new file mode 100755 index 000000000..abfe2a398 --- /dev/null +++ b/modules/modelarts/modelarts.go @@ -0,0 +1,65 @@ +package modelarts + +import ( + "code.gitea.io/gitea/modules/setting" + "path" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +const ( + storageTypeOBS = "obs" + autoStopDuration = 4 * 60 *60 + flavor = "modelarts.kat1.xlarge" + profileID = "Python3-ascend910-arm" + + subTaskName = "task1" + + DataSetMountPath = "/home/ma-user/work" +) + +func GenerateTask(ctx *context.Context, jobName, uuid string) error { + dataActualPath := setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + jobResult, err := CreateJob(models.CreateNotebookParams{ + JobName: jobName, + Description:"", + ProfileID: profileID, + Flavor: flavor, + Spec: models.Spec{ + Storage: models.Storage{ + Type: storageTypeOBS, + Location:models.Location{ + Path: dataActualPath, + }, + }, + AutoStop: models.AutoStop{ + Enable: true, + Duration: autoStopDuration, + }, + }, + + }) + if err != nil { + log.Error("CreateJob failed:", err.Error()) + return err + } + + err = models.CreateCloudbrain(&models.Cloudbrain{ + Status: string(models.JobWaiting), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobResult.ID, + JobName: jobName, + SubTaskName: subTaskName, + JobType: string(models.JobTypeDebug), + Type: models.TypeCloudBrainTwo, + }) + + if err != nil { + return err + } + + return nil +} diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go new file mode 100755 index 000000000..97a7bfa93 --- /dev/null +++ b/modules/modelarts/resty.go @@ -0,0 +1,240 @@ +package modelarts + +import ( + "code.gitea.io/gitea/modules/log" + "fmt" + "net/http" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + "github.com/go-resty/resty/v2" +) + +var ( + restyClient *resty.Client + HOST string + TOKEN string +) + +const ( + methodPassword = "password" + + urlGetToken = "/v3/auth/tokens" + urlNotebook = "/demanager/instances" + urlQuaryNotebook = "/demanager/instances" +) +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + } + return restyClient +} + +func checkSetting() { + if len(HOST) != 0 && len(TOKEN) != 0 && restyClient != nil { + return + } + + getToken() +} + +func getToken() error { + HOST = setting.ModelArtsHost + + client := getRestyClient() + params := models.GetTokenParams{ + Auth: models.Auth{ + Identity: models.Identity{ + Methods: []string{methodPassword}, + Password: models.Password{ + User: models.NotebookUser{ + Name: setting.ModelArtsUsername, + Password: setting.ModelArtsPassword, + Domain: models.Domain{ + Name: setting.ModelArtsDomain, + }, + }, + }, + }, + Scope: models.Scope{ + Project: models.Project{ + Name: setting.ProjectName, + }, + }, + }, + } + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(params). + Post(HOST + urlGetToken) + if err != nil { + return fmt.Errorf("resty getToken: %s", err) + } + + if res.StatusCode() != http.StatusCreated { + return fmt.Errorf("getToken failed:%s", res.String()) + } + + TOKEN = res.Header().Get("X-Subject-Token") + + return nil +} + +func CreateJob(createJobParams models.CreateNotebookParams) (*models.CreateNotebookResult, error) { + checkSetting() + client := getRestyClient() + var result models.CreateNotebookResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(createJobParams). + SetResult(&result). + Post(HOST + "/v1/" + setting.ProjectID + urlNotebook) + + if err != nil { + return nil, fmt.Errorf("resty create job: %s", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if len(result.ErrorCode) != 0 { + log.Error("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return &result, fmt.Errorf("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func GetJob(jobID string) (*models.GetNotebookResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetNotebookResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlNotebook + jobID) + + if err != nil { + return nil, fmt.Errorf("resty GetJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if len(result.ErrorCode) != 0 { + return &result, fmt.Errorf("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func GetImages() (*models.GetImagesResult, error) { + checkSetting() + client := getRestyClient() + var getImagesResult models.GetImagesResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&getImagesResult). + Get(HOST + "/rest-server/api/v1/image/list/") + + if err != nil { + return nil, fmt.Errorf("resty GetImages: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + //if len(result.ErrorCode) != 0 { + // return &result, fmt.Errorf("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + //} + + return &getImagesResult, nil +} + +func CommitImage(jobID string, params models.CommitImageParams) error { + checkSetting() + client := getRestyClient() + var result models.CommitImageResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(params). + SetResult(&result). + Post(HOST + "/rest-server/api/v1/jobs/" + jobID + "/commitImage") + + if err != nil { + return fmt.Errorf("resty CommitImage: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + //if len(result.ErrorCode) != 0 { + // return &result, fmt.Errorf("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + //} + + return nil +} + +func StopJob(jobID string) error { + checkSetting() + client := getRestyClient() + var result models.StopJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/rest-server/api/v1/jobs/" + jobID) + + if err != nil { + return fmt.Errorf("resty StopJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + //if len(result.ErrorCode) != 0 { + // return &result, fmt.Errorf("CreateJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + //} + + return nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index cf82578f3..3d8d19498 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -456,6 +456,14 @@ var ( Location string BasePath string //RealPath string + + //modelarts config + ModelArtsHost string + ProjectID string + ProjectName string + ModelArtsUsername string + ModelArtsPassword string + ModelArtsDomain string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1148,6 +1156,14 @@ func NewContext() { Bucket = sec.Key("BUCKET").MustString("testopendata") Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") + + sec = Cfg.Section("modelarts") + ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") + ProjectID = sec.Key("PROJECT_ID").MustString("") + ProjectName = sec.Key("PROJECT_NAME").MustString("") + ModelArtsUsername = sec.Key("USERNAME").MustString("") + ModelArtsPassword = sec.Key("PASSWORD").MustString("") + ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") } func loadInternalToken(sec *ini.Section) string { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5b19166be..7ebc0a44c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -46,7 +46,7 @@ func CloudBrainIndex(ctx *context.Context) { PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - // SortType: sortType, + Type: models.TypeCloudBrainOne, }) if err != nil { ctx.ServerError("Cloudbrain", err) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go new file mode 100755 index 000000000..ffa51beef --- /dev/null +++ b/routers/repo/modelarts.go @@ -0,0 +1,250 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/modelarts" + "errors" + "strconv" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/auth" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +const ( + tplModelArtsIndex base.TplName = "repo/modelarts/index" + tplModelArtsNew base.TplName = "repo/modelarts/new" + tplModelArtsShow base.TplName = "repo/modelarts/show" +) + +// MustEnableDataset check if repository enable internal cb +func MustEnableModelArts(ctx *context.Context) { + if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { + ctx.NotFound("MustEnableCloudbrain", nil) + return + } +} +func ModelArtsIndex(ctx *context.Context) { + MustEnableCloudbrain(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + // SortType: sortType, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + timestamp := time.Now().Unix() + for i, task := range ciTasks { + if task.Status == string(models.JobRunning) && (timestamp-int64(task.CreatedUnix) > 30) { + ciTasks[i].CanDebug = true + } else { + ciTasks[i].CanDebug = false + } + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = ciTasks + ctx.HTML(200, tplCloudBrainIndex) +} + +func ModelArtsNew(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + //attachs, err := models.GetAllUserAttachments(ctx.User.ID) + //if err != nil { + // ctx.ServerError("GetAllUserAttachments failed:", err) + // return + //} + + //ctx.Data["attachments"] = attachs + //ctx.Data["command"] = cloudbrain.Command + //ctx.Data["code_path"] = cloudbrain.CodeMountPath + ctx.Data["dataset_path"] = modelarts.DataSetMountPath + //ctx.Data["model_path"] = cloudbrain.ModelMountPath + ctx.HTML(200, tplModelArtsNew) +} + +func ModelArtsCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { + ctx.Data["PageIsCloudBrain"] = true + jobName := form.JobName + uuid := form.Attachment + //repo := ctx.Repo.Repository + + err := modelarts.GenerateTask(ctx, jobName, uuid) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts") +} + +func ModelArtsShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + } + + //result, err := modelarts.GetJob(jobID) + result, err := cloudbrain.GetJob(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + ctx.Data["result"] = jobRes + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + ctx.Data["taskRes"] = taskRes + task.Status = taskRes.TaskStatuses[0].State + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + } + } + + ctx.Data["task"] = task + ctx.Data["jobID"] = jobID + ctx.HTML(200, tplCloudBrainShow) +} + +func ModelArtsDebug(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + //https://console.ai.pcl.cn/modelarts/internal/hub/notebook/user/DE-afcdf674-6489-11eb-bfe7-0255ac100057/lab + //debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + debugUrl := "https://console.ai.pcl.cn/modelarts/internal/hub/notebook/user/" + task.JobID + "/lab" + ctx.Redirect(debugUrl) +} + +func ModelArtsCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": "GetCloudbrainByJobID failed", + }) + return + } + + err = cloudbrain.CommitImage(jobID, models.CommitImageParams{ + Ip: task.ContainerIp, + TaskContainerId: task.ContainerID, + ImageDescription: form.Description, + ImageTag: form.Tag, + }) + if err != nil { + log.Error("CommitImage(%s) failed:", task.JobName, err.Error()) + ctx.JSON(200, map[string]string{ + "result_code": "-1", + "error_msg": "CommitImage failed", + }) + return + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + "error_msg": "", + }) +} + +func ModelArtsStop(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + log.Info(jobID) + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobRunning) { + log.Error("the job(%s) is not running", task.JobName) + ctx.ServerError("the job is not running", errors.New("the job is not running")) + return + } + + err = cloudbrain.StopJob(jobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("StopJob failed", err) + return + } + + task.Status = string(models.JobStopped) + err = models.UpdateJob(task) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") +} + +func ModelArtsDel(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobStopped) { + log.Error("the job(%s) has not been stopped", task.JobName) + ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) + return + } + + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") +} + +func ModelArtsBenchmark(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + _, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + ctx.Redirect(setting.BenchmarkServerHost) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a49dd4b65..9499890bd 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -915,6 +915,18 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) + m.Group("/modelarts", func() { + m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow) + m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug) + //m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) + //m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) + }) + m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate) + }, context.RepoRef()) + m.Group("/blockchain", func() { m.Get("", repo.BlockChainIndex) }, context.RepoRef())