Browse Source

Merge branch 'V20220718' into phone

pull/2447/head
ychao_1983 2 years ago
parent
commit
b1e95c666f
74 changed files with 10335 additions and 3969 deletions
  1. +6
    -0
      custom/conf/app.ini.sample
  2. +10
    -8
      models/action.go
  3. +38
    -1
      models/attachment.go
  4. +181
    -2
      models/cloudbrain.go
  5. +5
    -0
      models/cloudbrain_image.go
  6. +2
    -6
      models/custom_migrations.go
  7. +136
    -13
      models/dataset.go
  8. +7
    -0
      models/dataset_star.go
  9. +2
    -7
      models/repo.go
  10. +50
    -21
      models/user_business_analysis.go
  11. +8
    -6
      models/user_business_struct.go
  12. +26
    -0
      modules/auth/grampus.go
  13. +139
    -0
      modules/auth/wechat/auto_reply.go
  14. +39
    -2
      modules/auth/wechat/client.go
  15. +97
    -4
      modules/auth/wechat/event_handle.go
  16. +13
    -0
      modules/auth/wechat/material.go
  17. +184
    -144
      modules/cloudbrain/cloudbrain.go
  18. +217
    -0
      modules/grampus/grampus.go
  19. +277
    -0
      modules/grampus/resty.go
  20. +6
    -14
      modules/modelarts/modelarts.go
  21. +35
    -0
      modules/setting/setting.go
  22. +17
    -8
      modules/setting/webhook.go
  23. +10
    -0
      modules/util/path.go
  24. +1
    -1
      modules/util/util.go
  25. +53
    -4
      modules/webhook/deliver.go
  26. +14
    -0
      options/locale/locale_en-US.ini
  27. +16
    -4
      options/locale/locale_zh-CN.ini
  28. +31
    -18
      public/home/home.js
  29. +5
    -3
      public/home/search.js
  30. +2
    -7
      routers/admin/cloudbrains.go
  31. +4
    -3
      routers/admin/dataset.go
  32. +10
    -0
      routers/api/v1/api.go
  33. +3
    -3
      routers/api/v1/repo/cloudbrain_dashboard.go
  34. +67
    -8
      routers/api/v1/repo/modelarts.go
  35. +3
    -10
      routers/api/v1/repo/repo_dashbord.go
  36. +22
    -0
      routers/authentication/wechat.go
  37. +129
    -15
      routers/authentication/wechat_event.go
  38. +13
    -10
      routers/home.go
  39. +150
    -36
      routers/repo/cloudbrain.go
  40. +78
    -0
      routers/repo/dataset.go
  41. +843
    -0
      routers/repo/grampus.go
  42. +154
    -45
      routers/repo/modelarts.go
  43. +23
    -0
      routers/routes/routes.go
  44. +2
    -7
      routers/user/home.go
  45. +1
    -0
      routers/user/profile.go
  46. +1
    -1
      services/socketwrap/clientManager.go
  47. +3
    -3
      templates/admin/cloudbrain/list.tmpl
  48. +4
    -4
      templates/custom/select_dataset.tmpl
  49. +4
    -4
      templates/custom/select_dataset_train.tmpl
  50. +1
    -1
      templates/explore/datasets.tmpl
  51. +1
    -1
      templates/repo/cloudbrain/benchmark/show.tmpl
  52. +5
    -5
      templates/repo/cloudbrain/new.tmpl
  53. +1
    -1
      templates/repo/cloudbrain/show.tmpl
  54. +59
    -48
      templates/repo/cloudbrain/trainjob/new.tmpl
  55. +4
    -0
      templates/repo/cloudbrain/trainjob/show.tmpl
  56. +440
    -0
      templates/repo/grampus/trainjob/gpu/new.tmpl
  57. +431
    -0
      templates/repo/grampus/trainjob/npu/new.tmpl
  58. +982
    -0
      templates/repo/grampus/trainjob/show.tmpl
  59. +4
    -1
      templates/repo/modelarts/notebook/new.tmpl
  60. +32
    -44
      templates/repo/modelarts/notebook/show.tmpl
  61. +4
    -4
      templates/repo/modelarts/trainjob/index.tmpl
  62. +134
    -133
      templates/repo/modelarts/trainjob/new.tmpl
  63. +5
    -0
      templates/repo/modelarts/trainjob/show.tmpl
  64. +3
    -3
      templates/user/dashboard/cloudbrains.tmpl
  65. +7
    -3
      templates/user/dashboard/feeds.tmpl
  66. +1155
    -0
      web_src/js/components/dataset/selectDataset.vue
  67. +1
    -1
      web_src/js/components/images/Images.vue
  68. +292
    -0
      web_src/js/components/images/selectGrampusImages.vue
  69. +560
    -367
      web_src/js/components/images/selectImages.vue
  70. +13
    -0
      web_src/js/features/images.js
  71. +91
    -65
      web_src/js/features/letteravatar.js
  72. +2117
    -2142
      web_src/js/index.js
  73. +3
    -0
      web_src/less/_base.less
  74. +849
    -728
      web_src/less/openi.less

+ 6
- 0
custom/conf/app.ini.sample View File

@@ -1141,3 +1141,9 @@ growth_issue=0.2
growth_contributors=0.2
growth_commit=0.2
growth_comments=0.2


[grampus]
USERNAME =
PASSWORD =
SERVER_HOST =

+ 10
- 8
models/action.go View File

@@ -50,14 +50,16 @@ const (
ActionRejectPullRequest // 22
ActionCommentPull // 23

ActionUploadAttachment //24
ActionCreateDebugGPUTask //25
ActionCreateDebugNPUTask //26
ActionCreateTrainTask //27
ActionCreateInferenceTask // 28
ActionCreateBenchMarkTask //29
ActionCreateNewModelTask //30
ActionCreateGPUTrainTask //31
ActionUploadAttachment //24
ActionCreateDebugGPUTask //25
ActionCreateDebugNPUTask //26
ActionCreateTrainTask //27
ActionCreateInferenceTask // 28
ActionCreateBenchMarkTask //29
ActionCreateNewModelTask //30
ActionCreateGPUTrainTask //31
ActionCreateGrampusNPUTrainTask //32
ActionCreateGrampusGPUTrainTask //33
)

// Action represents user operation type and other information to


+ 38
- 1
models/attachment.go View File

@@ -110,8 +110,15 @@ func (a *Attachment) IncreaseDownloadCount() error {
}

func IncreaseAttachmentUseNumber(uuid string) error {

uuidArray := strings.Split(uuid, ";")
for i := range uuidArray {
uuidArray[i] = "'" + uuidArray[i] + "'"
}

uuidInCondition := "(" + strings.Join(uuidArray, ",") + ")"
// Update use number.
if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid=?", uuid); err != nil {
if _, err := x.Exec("UPDATE `attachment` SET use_number=use_number+1 WHERE uuid in " + uuidInCondition); err != nil {
return fmt.Errorf("increase attachment use count: %v", err)
}

@@ -560,6 +567,36 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
return total, nil
}

func AttachmentsByDatasetOption(datasets []int64, opts *SearchDatasetOptions) ([]*Attachment, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(builder.In("attachment.dataset_id", datasets))
if opts.JustNeedZipFile {
cond = cond.And(builder.Gt{"attachment.decompress_state": 0})
}
if opts.PublicOnly {
cond = cond.And(builder.Eq{"attachment.is_private": false})
}
if opts.CloudBrainType >= 0 {
cond = cond.And(builder.Eq{"attachment.type": opts.CloudBrainType})
}
if opts.UploadAttachmentByMe {
cond = cond.And(
builder.Eq{"attachment.uploader_id": opts.User.ID},
)
}


attachments := make([]*Attachment, 0)
if err := sess.Table(&Attachment{}).Where(cond).Desc("id").
Find(&attachments); err != nil {
return nil, fmt.Errorf("Find: %v", err)
}
return attachments, nil

}

func GetAllAttachmentSize() (int64, error) {
return x.SumInt(&Attachment{}, "size")
}


+ 181
- 2
models/cloudbrain.go View File

@@ -2,6 +2,7 @@ package models

import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
@@ -24,7 +25,7 @@ type ModelArtsJobStatus string
const (
TypeCloudBrainOne int = iota
TypeCloudBrainTwo
TypeIntelligentNet
TypeC2Net //智算网络

TypeCloudBrainAll = -1
)
@@ -99,6 +100,15 @@ const (
ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败

DURATION_STR_ZERO = "00:00:00"

//grampus
GrampusStatusPending = "pending"
GrampusStatusRunning = "RUNNING"
GrampusStatusFailed = "FAILED"
GrampusStatusSucceeded = "SUCCEEDED"
GrampusStatusStopped = "STOPPED"
GrampusStatusUnknown = "UNKNOWN"
GrampusStatusWaiting = "WAITING"
)

type Cloudbrain struct {
@@ -138,6 +148,8 @@ type Cloudbrain struct {
PreVersionName string //父版本名称
ComputeResource string //计算资源,例如npu
EngineID int64 //引擎id
ImageID string //grampus image_id
AiCenter string //grampus ai center: center_id+center_name

TrainUrl string //输出模型的obs路径
BranchName string //分支名称
@@ -206,7 +218,7 @@ func ConvertDurationToStr(duration int64) string {
}

func IsTrainJobTerminal(status string) bool {
return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled)
return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled) || status == GrampusStatusFailed || status == GrampusStatusStopped || status == GrampusStatusSucceeded
}

func IsModelArtsDebugJobTerminal(status string) bool {
@@ -554,6 +566,17 @@ type FlavorInfo struct {
Desc string `json:"desc"`
}

type SpecialPools struct {
Pools []*SpecialPool `json:"pools"`
}
type SpecialPool struct {
Org string `json:"org"`
Type string `json:"type"`
IsExclusive bool `json:"isExclusive"`
Pool []*GpuInfo `json:"pool"`
JobType []string `json:"jobType"`
}

type ImageInfosModelArts struct {
ImageInfo []*ImageInfoModelArts `json:"image_info"`
}
@@ -977,6 +1000,16 @@ type Parameter struct {
type Parameters struct {
Parameter []Parameter `json:"parameter"`
}
type Datasurl struct {
DatasetUrl string `json:"dataset_url"`
DatasetName string `json:"dataset_name"`
}

type DatasetDownload struct {
DatasetName string `json:"dataset_name"`
DatasetDownloadLink string `json:"dataset_download_link"`
RepositoryLink string `json:"repository_link"`
}

type DataSource struct {
DatasetID string `json:"dataset_id"`
@@ -1156,6 +1189,88 @@ type LogFile struct {
Name string
}

//Grampus
type GrampusResult struct {
ErrorCode int `json:"errorCode"`
ErrorMsg string `json:"errorMsg"`
}

type GrampusJobInfo struct {
StartedAt int64 `json:"startedAt"`
RunSec int64 `json:"runSec"`
CompletedAt int64 `json:"completedAt"`
CreatedAt int64 `json:"createdAt"`
UpdatedAt int64 `json:"updatedAt"`
Desc string `json:"desc"`
JobID string `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
UserID string `json:"userId"`
Tasks []GrampusTasks `json:"tasks"`
}
type Center struct {
ID string `json:"id"`
Name string `json:"name"`
}
type GrampusSpec struct {
CreatedAt int64 `json:"createdAt"`
UpdatedAt int64 `json:"updatedAt"`
ID string `json:"id"`
Name string `json:"name"`
ProcessorType string `json:"processorType"`
Centers []Center `json:"centers"`
}

type GetGrampusResourceSpecsResult struct {
GrampusResult
Infos []GrampusSpec `json:"resourceSpecs"`
}

type GrampusImage struct {
CreatedAt int64 `json:"createdAt"`
UpdatedAt int64 `json:"updatedAt"`
ID string `json:"id"`
Name string `json:"name"`
ProcessorType string `json:"processorType"`
}

type GetGrampusImagesResult struct {
GrampusResult
TotalSize int `json:"totalSize"`
Infos []GrampusImage `json:"images"`
}

type CreateGrampusJobResponse struct {
GrampusResult
JobInfo GrampusJobInfo `json:"otJob"`
}

type GetGrampusJobResponse struct {
GrampusResult
JobInfo GrampusJobInfo `json:"otJob"`
}

type GrampusStopJobResponse struct {
GrampusResult
StoppedAt int64 `json:"stoppedAt"`
}

type GrampusTasks struct {
Command string `json:"command"`
Name string `json:"name"`
ImageId string `json:"imageId"`
ResourceSpecId string `json:"resourceSpecId"`
ImageUrl string `json:"imageUrl"`
CenterID []string `json:"centerID"`
CenterName []string `json:"centerName"`
ReplicaNum int `json:"replicaNum"`
}

type CreateGrampusJobRequest struct {
Name string `json:"name"`
Tasks []GrampusTasks `json:"tasks"`
}

type GetTrainJobMetricStatisticResult struct {
TrainJobResult
Interval int `json:"interval"` //查询的时间间隔,单位为分钟
@@ -1201,6 +1316,12 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
)
}

if len(opts.ComputeResource) > 0 {
cond = cond.And(
builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource},
)
}

if len(opts.JobTypes) > 0 {
if opts.JobTypeNot {
cond = cond.And(
@@ -1456,6 +1577,11 @@ func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) {
return getRepoCloudBrain(cb)
}

func GetCloudbrainByJobIDWithDeleted(jobID string) (*Cloudbrain, error) {
cb := &Cloudbrain{JobID: jobID}
return getRepoCloudBrainWithDeleted(cb)
}

func GetCloudbrainByID(id string) (*Cloudbrain, error) {
idInt64, _ := strconv.ParseInt(id, 10, 64)
cb := &Cloudbrain{ID: idInt64}
@@ -1634,6 +1760,11 @@ func GetCloudbrainInferenceJobCountByUserID(userID int64) (int, error) {
return int(count), err
}

func GetGrampusCountByUserID(userID int64, jobType, computeResource string) (int, error) {
count, err := x.In("status", GrampusStatusWaiting, GrampusStatusRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeC2Net).And("compute_resource = ?", computeResource).Count(new(Cloudbrain))
return int(count), err
}

func UpdateInferenceJob(job *Cloudbrain) error {
return updateInferenceJob(x, job)
}
@@ -1839,3 +1970,51 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er
}
return cloudbrains, count, nil
}

type DatasetInfo struct {
DataLocalPath string
Name string
}

func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) {
var datasetNames string
uuids := strings.Split(uuidStr, ";")
if len(uuids) > setting.MaxDatasetNum {
log.Error("the dataset count(%d) exceed the limit", len(uuids))
return nil, datasetNames, errors.New("the dataset count exceed the limit")
}

datasetInfos := make(map[string]DatasetInfo)
attachs, err := GetAttachmentsByUUIDs(uuids)
if err != nil {
log.Error("GetAttachmentsByUUIDs failed: %v", err)
return nil, datasetNames, err
}
for i, attach := range attachs {
fileName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(attach.Name, ".zip"), ".tar.gz"), ".tgz")
for _, datasetInfo := range datasetInfos {
if fileName == datasetInfo.Name {
log.Error("the dataset name is same: %v", attach.Name)
return nil, datasetNames, errors.New("the dataset name is same")
}
}

dataLocalPath := setting.Attachment.Minio.RealPath +
setting.Attachment.Minio.Bucket + "/" +
setting.Attachment.Minio.BasePath +
AttachmentRelativePath(attach.UUID) +
attach.UUID

datasetInfos[attach.UUID] = DatasetInfo{
DataLocalPath: dataLocalPath,
Name: fileName,
}
if i == 0 {
datasetNames = attach.Name
} else {
datasetNames += ";" + attach.Name
}
}

return datasetInfos, datasetNames, nil
}

+ 5
- 0
models/cloudbrain_image.go View File

@@ -68,6 +68,7 @@ type SearchImageOptions struct {
IncludeCustom bool
IncludeOwnerOnly bool
Topics string
CloudbrainType int
ListOptions
SearchOrderBy
}
@@ -411,6 +412,10 @@ func SearchImageCondition(opts *SearchImageOptions) builder.Cond {

}

if opts.CloudbrainType > 0 {
cond = cond.And(builder.Eq{"cloudbrain_type": opts.CloudbrainType})
}

return cond
}



+ 2
- 6
models/custom_migrations.go View File

@@ -15,13 +15,9 @@ type CustomMigrationStatic struct {
Migrate func(*xorm.Engine, *xorm.Engine) error
}

var customMigrations = []CustomMigration{
{"Custom v1 Topic struct change to support chinese", syncTopicStruct},
}
var customMigrations []CustomMigration

var customMigrationsStatic = []CustomMigrationStatic{
{"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate},
}
var customMigrationsStatic []CustomMigrationStatic

func MigrateCustom(x *xorm.Engine) {



+ 136
- 13
models/dataset.go View File

@@ -81,12 +81,14 @@ func (datasets DatasetList) loadAttributes(e Engine) error {
if err := e.
Where("id > 0").
In("id", keysInt64(userIdSet)).
Cols("id", "lower_name", "name", "full_name", "email").
Find(&users); err != nil {
return fmt.Errorf("find users: %v", err)
}
if err := e.
Where("id > 0").
In("id", keysInt64(set)).
Cols("id", "owner_id", "owner_name", "lower_name", "name", "description", "alias", "lower_alias","is_private").
Find(&repos); err != nil {
return fmt.Errorf("find repos: %v", err)
}
@@ -98,19 +100,94 @@ func (datasets DatasetList) loadAttributes(e Engine) error {
return nil
}

func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) error {
if len(datasets) == 0 {
return nil
}
datasetIDs := make([]int64, len(datasets))
for i := range datasets {
datasetIDs[i] = datasets[i].ID
}
attachments, err := AttachmentsByDatasetOption(datasetIDs, opts)
if err != nil {
return fmt.Errorf("GetAttachmentsByDatasetIds failed error: %v", err)
}

permissionMap := make(map[int64]bool, len(datasets))

for _, attachment := range attachments {

for i := range datasets {
if attachment.DatasetID == datasets[i].ID {
if opts.StarByMe {

permission,ok := permissionMap[datasets[i].ID];
if !ok {

permission = false
datasets[i].Repo.GetOwner()
if datasets[i].Repo.Owner.IsOrganization() {
if datasets[i].Repo.Owner.IsUserPartOfOrg(opts.User.ID) {
log.Info("user is member of org.")
permission = true
}
}
if !permission {
isCollaborator, _ := datasets[i].Repo.IsCollaborator(opts.User.ID)
if isCollaborator {
log.Info("Collaborator user may visit the attach.")
permission = true
}
}

permissionMap[datasets[i].ID]=permission
}

if permission{
datasets[i].Attachments = append(datasets[i].Attachments, attachment)
} else if !attachment.IsPrivate {
datasets[i].Attachments = append(datasets[i].Attachments, attachment)
}
} else {
datasets[i].Attachments = append(datasets[i].Attachments, attachment)
}

}

}

}

for i := range datasets {
if datasets[i].Attachments==nil{
datasets[i].Attachments=[]*Attachment{}
}
datasets[i].Repo.Owner = nil
}
return nil

}

type SearchDatasetOptions struct {
Keyword string
OwnerID int64
User *User
RepoID int64
IncludePublic bool
RecommendOnly bool
Category string
Task string
License string
DatasetIDs []int64
DatasetIDs []int64 // 目前只在StarByMe为true时起作用
ListOptions
SearchOrderBy
IsOwner bool
IsOwner bool
StarByMe bool
CloudBrainType int //0 cloudbrain 1 modelarts -1 all
PublicOnly bool
JustNeedZipFile bool
NeedAttachment bool
UploadAttachmentByMe bool
}

func CreateDataset(dataset *Dataset) (err error) {
@@ -159,29 +236,40 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond {
if opts.RepoID > 0 {
cond = cond.And(builder.Eq{"dataset.repo_id": opts.RepoID})
}
if opts.IncludePublic {

if opts.PublicOnly {
cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic})
cond = cond.And(builder.Eq{"attachment.is_private": false})
} else if opts.IncludePublic {
cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic})
cond = cond.And(builder.Eq{"attachment.is_private": false})
if opts.OwnerID > 0 {

subCon := builder.NewCond()
subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID})
subCon = generateFilterCond(opts, subCon)
cond = cond.Or(subCon)

}
} else if opts.OwnerID > 0 {
} else if opts.OwnerID > 0 && !opts.StarByMe && !opts.UploadAttachmentByMe {
cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID})
if !opts.IsOwner {
cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic})
cond = cond.And(builder.Eq{"attachment.is_private": false})
}
}

if len(opts.DatasetIDs) > 0 {
subCon := builder.NewCond()
subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs))
cond = cond.Or(subCon)
if opts.StarByMe {
cond = cond.And(builder.In("dataset.id", opts.DatasetIDs))
} else {
subCon := builder.NewCond()
subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs))
subCon = generateFilterCond(opts, subCon)
cond = cond.Or(subCon)
}
} else {
if opts.StarByMe {
cond = cond.And(builder.Eq{"dataset.id": -1})
}
}

return cond
@@ -207,6 +295,17 @@ func generateFilterCond(opts *SearchDatasetOptions, cond builder.Cond) builder.C
cond = cond.And(builder.Eq{"dataset.recommend": opts.RecommendOnly})
}

if opts.JustNeedZipFile {
cond = cond.And(builder.Gt{"attachment.decompress_state": 0})
}

if opts.CloudBrainType >= 0 {
cond = cond.And(builder.Eq{"attachment.type": opts.CloudBrainType})
}
if opts.UploadAttachmentByMe {
cond = cond.And(builder.Eq{"attachment.uploader_id": opts.User.ID})
}

return cond
}

@@ -233,7 +332,6 @@ func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (Da
sess.Select(selectColumnsSql).Join("INNER", "repository", "repository.id = dataset.repo_id").
Join("INNER", "attachment", "attachment.dataset_id=dataset.id").
Where(cond).OrderBy(opts.SearchOrderBy.String())

if opts.PageSize > 0 {
sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize)
}
@@ -245,6 +343,12 @@ func SearchDatasetByCondition(opts *SearchDatasetOptions, cond builder.Cond) (Da
return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}

if opts.NeedAttachment {
if err = datasets.loadAttachmentAttributes(opts); err != nil {
return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}
}

return datasets, count, nil
}

@@ -361,10 +465,22 @@ func UpdateDataset(ctx DBContext, rel *Dataset) error {
func IncreaseDatasetUseCount(uuid string) {

IncreaseAttachmentUseNumber(uuid)
attachments, _ := GetAttachmentsByUUIDs(strings.Split(uuid, ";"))

countMap := make(map[int64]int)

for _, attachment := range attachments {
value, ok := countMap[attachment.DatasetID]
if ok {
countMap[attachment.DatasetID] = value + 1
} else {
countMap[attachment.DatasetID] = 1
}

}

attachment, _ := GetAttachmentByUUID(uuid)
if attachment != nil {
x.Exec("UPDATE `dataset` SET use_count=use_count+1 WHERE id=?", attachment.DatasetID)
for key, value := range countMap {
x.Exec("UPDATE `dataset` SET use_count=use_count+? WHERE id=?", value, key)
}

}
@@ -460,5 +576,12 @@ func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 {
_ = x.Table("dataset").Join("INNER", "collaboration", "dataset.repo_id = collaboration.repo_id and collaboration.mode>0 and collaboration.user_id=?", userID).
Cols("dataset.id").Find(&datasets)
return datasets
}

func GetTeamDatasetIdsByUserID(userID int64) []int64 {
var datasets []int64
_ = x.Table("dataset").Join("INNER", "team_repo", "dataset.repo_id = team_repo.repo_id").
Join("INNER", "team_user", "team_repo.team_id=team_user.team_id and team_user.uid=?", userID).
Cols("dataset.id").Find(&datasets)
return datasets
}

+ 7
- 0
models/dataset_star.go View File

@@ -68,3 +68,10 @@ func isDatasetStaring(e Engine, userID, datasetID int64) bool {
has, _ := e.Get(&DatasetStar{0, userID, datasetID, 0})
return has
}

func GetDatasetIdsStarByUser(userID int64) []int64 {
var datasets []int64
_ = x.Table("dataset_star").Where("uid=?", userID).
Cols("dataset_star.dataset_id").Find(&datasets)
return datasets
}

+ 2
- 7
models/repo.go View File

@@ -2749,15 +2749,10 @@ func ReadLatestFileInRepo(userName, repoName, refName, treePath string) (*RepoFi
log.Error("ReadLatestFileInRepo: Close: %v", err)
}
}()

buf := make([]byte, 1024)
n, _ := reader.Read(buf)
if n >= 0 {
buf = buf[:n]
}
d, _ := ioutil.ReadAll(reader)
commitId := ""
if blob != nil {
commitId = fmt.Sprint(blob.ID)
}
return &RepoFile{CommitId: commitId, Content: buf}, nil
return &RepoFile{CommitId: commitId, Content: d}, nil
}

+ 50
- 21
models/user_business_analysis.go View File

@@ -955,6 +955,8 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
return err
}
userNewAddActivity := make(map[int64]map[int64]int64)
userAcitvateJsonMap := make(map[int64]map[int64]int64)
userCurrentDayRegistMap := make(map[int64]map[int64]int64)
ParaWeight := getParaWeight()
userMetrics := make(map[string]int)
var indexTotal int64
@@ -1028,7 +1030,10 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
log.Info("has activity." + userRecord.Name)
addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID)
}

if userRecord.IsActive {
addUserToMap(userAcitvateJsonMap, userRecord.CreatedUnix, dateRecord.ID)
}
addUserToMap(userCurrentDayRegistMap, userRecord.CreatedUnix, dateRecord.ID)
}

indexTotal += PAGE_SIZE
@@ -1064,36 +1069,61 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
}
statictisSess.Insert(&useMetrics)
//update new user activity
updateNewUserAcitivity(userNewAddActivity, statictisSess)
updateNewUserAcitivity(userNewAddActivity, userAcitvateJsonMap, userCurrentDayRegistMap, statictisSess)
return nil
}

func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) {
for key, value := range currentUserActivity {
func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, userAcitvateJsonMap map[int64]map[int64]int64, userCurrentDayRegistMap map[int64]map[int64]int64, statictisSess *xorm.Session) {
for key, value := range userCurrentDayRegistMap {
useMetrics := &UserMetrics{CountDate: key}
userAcitvateValue := userAcitvateJsonMap[key]
HuodongValue := currentUserActivity[key]
has, err := statictisSess.Get(useMetrics)
if err == nil && has {
userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",")
for _, userIdStr := range userIdArrays {
userIdInt, err := strconv.ParseInt(userIdStr, 10, 64)
if err == nil {
value[userIdInt] = userIdInt
}
}
userIdArray := ""
for _, tmpValue := range value {
userIdArray += fmt.Sprint(tmpValue) + ","
}
useMetrics.HasActivityUser = len(value)
if len(userIdArray) > 0 {
useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1]
}
updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson + "',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key)
ActivityUserArray, HuodongTotal := setUniqueUserId(useMetrics.HasActivityUserJson, HuodongValue)
useMetrics.HasActivityUser = HuodongTotal
useMetrics.HasActivityUserJson = ActivityUserArray

useMetrics.CurrentDayRegistUser = len(value)

RegistUserArray, lenRegistUser := setUniqueUserId(useMetrics.ActivityUserJson, userAcitvateValue)
useMetrics.ActivityUserJson = RegistUserArray
useMetrics.ActivateRegistUser = lenRegistUser

updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson +
"',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) +
",activity_user_json='" + useMetrics.ActivityUserJson + "'" +
",activate_regist_user=" + fmt.Sprint(useMetrics.ActivateRegistUser) +
",not_activate_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser-useMetrics.ActivateRegistUser) +
",current_day_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser) +
" where count_date=" + fmt.Sprint(key)

statictisSess.Exec(updateSql)
}
}
}

func setUniqueUserId(jsonString string, value map[int64]int64) (string, int) {
if value == nil {
value = make(map[int64]int64, 0)
}
userIdArrays := strings.Split(jsonString, ",")
for _, userIdStr := range userIdArrays {
userIdInt, err := strconv.ParseInt(userIdStr, 10, 64)
if err == nil {
value[userIdInt] = userIdInt
}
}
userIdArray := ""
for _, tmpValue := range value {
userIdArray += fmt.Sprint(tmpValue) + ","
}
if len(userIdArray) > 0 {
return userIdArray[0 : len(userIdArray)-1], len(value)
}
return userIdArray, len(value)
}

func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) {
CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location())
CountDate := CountDateTime.Unix()
@@ -1104,7 +1134,6 @@ func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate time
} else {
currentUserActivity[CountDate][userId] = userId
}

}

func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) {


+ 8
- 6
models/user_business_struct.go View File

@@ -467,11 +467,11 @@ type UserAnalysisPara struct {

type UserMetrics struct {
CountDate int64 `xorm:"pk"`
ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"`
NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"`
ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"`
RegistActivityUser int `xorm:"NOT NULL DEFAULT 0"`
HasActivityUser int `xorm:"NOT NULL DEFAULT 0"`
ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天激活用户
NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天未激活用户
ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"` //激活比率
RegistActivityUser int `xorm:"NOT NULL DEFAULT 0"` //当天注册激活的人中,有贡献活动的人
HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` //当天有贡献活动的人
TotalUser int `xorm:"NOT NULL DEFAULT 0"`
TotalRegistUser int `xorm:"-"`
TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"`
@@ -480,5 +480,7 @@ type UserMetrics struct {
DisplayDate string `xorm:"-"`
DataDate string `xorm:"NULL"`
DaysForMonth int `xorm:"NOT NULL DEFAULT 0"`
HasActivityUserJson string `xorm:"text NULL"`
HasActivityUserJson string `xorm:"text NULL"` //贡献活动用户列表
ActivityUserJson string `xorm:"text NULL"` //激活用户列表
CurrentDayRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天注册用户
}

+ 26
- 0
modules/auth/grampus.go View File

@@ -0,0 +1,26 @@
package auth

import (
"gitea.com/macaron/binding"
"gitea.com/macaron/macaron"
)

type CreateGrampusTrainJobForm struct {
DisplayJobName string `form:"display_job_name" binding:"Required"`
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment" binding:"Required"`
BootFile string `form:"boot_file" binding:"Required"`
ImageID string `form:"image_id" binding:"Required"`
FlavorID string `form:"flavor" binding:"Required"`
Params string `form:"run_para_list" binding:"Required"`
Description string `form:"description"`
BranchName string `form:"branch_name" binding:"Required"`
FlavorName string `form:"flavor_name" binding:"Required"`
EngineName string `form:"engine_name" binding:"Required"`
WorkServerNumber int `form:"work_server_number" binding:"Required"`
Image string `form:"image"`
}

func (f *CreateGrampusTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

+ 139
- 0
modules/auth/wechat/auto_reply.go View File

@@ -0,0 +1,139 @@
package wechat

import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"encoding/json"
"github.com/patrickmn/go-cache"
"strings"
"time"
)

var WechatReplyCache = cache.New(2*time.Minute, 1*time.Minute)

const (
WECHAT_REPLY_CACHE_KEY = "wechat_response"
)

const (
ReplyTypeText = "text"
ReplyTypeImage = "image"
ReplyTypeVoice = "voice"
ReplyTypeVideo = "video"
ReplyTypeMusic = "music"
ReplyTypeNews = "news"
)

type ReplyConfigType string

const (
SubscribeReply ReplyConfigType = "subscribe"
AutoMsgReply ReplyConfigType = "autoMsg"
)

func (r ReplyConfigType) Name() string {
switch r {
case SubscribeReply:
return "subscribe"
case AutoMsgReply:
return "autoMsg"
}
return ""
}

func (r ReplyConfigType) TreePath() string {
switch r {
case SubscribeReply:
return setting.TreePathOfSubscribe
case AutoMsgReply:
return setting.TreePathOfAutoMsgReply
}
return ""
}

type WechatReplyContent struct {
Reply *ReplyContent
ReplyType string
KeyWords []string
IsFullMatch int
}

type ReplyContent struct {
Content string
MediaId string
Title string
Description string
MusicUrl string
HQMusicUrl string
ThumbMediaId string
Articles []ArticlesContent
}

func GetAutomaticReply(msg string) *WechatReplyContent {
r, err := LoadReplyFromCacheAndDisk(AutoMsgReply)
if err != nil {
return nil
}
if r == nil || len(r) == 0 {
return nil
}
for i := 0; i < len(r); i++ {
if r[i].IsFullMatch == 0 {
for _, v := range r[i].KeyWords {
if strings.Contains(msg, v) {
return r[i]
}
}
} else if r[i].IsFullMatch > 0 {
for _, v := range r[i].KeyWords {
if msg == v {
return r[i]
}
}
}
}
return nil

}

func loadReplyFromDisk(replyConfig ReplyConfigType) ([]*WechatReplyContent, error) {
log.Info("LoadReply from disk")
repo, err := models.GetRepositoryByOwnerAndAlias(setting.UserNameOfWechatReply, setting.RepoNameOfWechatReply)
if err != nil {
log.Error("get AutomaticReply repo failed, error=%v", err)
return nil, err
}
repoFile, err := models.ReadLatestFileInRepo(setting.UserNameOfWechatReply, repo.Name, setting.RefNameOfWechatReply, replyConfig.TreePath())
if err != nil {
log.Error("get AutomaticReply failed, error=%v", err)
return nil, err
}
res := make([]*WechatReplyContent, 0)
json.Unmarshal(repoFile.Content, &res)
if res == nil || len(res) == 0 {
return nil, err
}
return res, nil
}

func LoadReplyFromCacheAndDisk(replyConfig ReplyConfigType) ([]*WechatReplyContent, error) {
v, success := WechatReplyCache.Get(replyConfig.Name())
if success {
log.Info("LoadReply from cache,value = %v", v)
if v == nil {
return nil, nil
}
n := v.([]*WechatReplyContent)
return n, nil
}

content, err := loadReplyFromDisk(replyConfig)
if err != nil {
log.Error("LoadReply failed, error=%v", err)
WechatReplyCache.Set(replyConfig.Name(), nil, 30*time.Second)
return nil, err
}
WechatReplyCache.Set(replyConfig.Name(), content, 60*time.Second)
return content, nil
}

+ 39
- 2
modules/auth/wechat/client.go View File

@@ -17,7 +17,8 @@ var (
const (
GRANT_TYPE = "client_credential"
ACCESS_TOKEN_PATH = "/cgi-bin/token"
QR_CODE_Path = "/cgi-bin/qrcode/create"
QR_CODE_PATH = "/cgi-bin/qrcode/create"
GET_MATERIAL_PATH = "/cgi-bin/material/batchget_material"
ACTION_QR_STR_SCENE = "QR_STR_SCENE"

ERR_CODE_ACCESSTOKEN_EXPIRE = 42001
@@ -40,6 +41,11 @@ type QRCodeRequest struct {
Action_info ActionInfo `json:"action_info"`
Expire_seconds int `json:"expire_seconds"`
}
type MaterialRequest struct {
Type string `json:"type"`
Offset int `json:"offset"`
Count int `json:"count"`
}

type ActionInfo struct {
Scene Scene `json:"scene"`
@@ -97,7 +103,7 @@ func callQRCodeCreate(sceneStr string) (*QRCodeResponse, bool) {
SetQueryParam("access_token", GetWechatAccessToken()).
SetBody(bodyJson).
SetResult(&result).
Post(setting.WechatApiHost + QR_CODE_Path)
Post(setting.WechatApiHost + QR_CODE_PATH)
if err != nil {
log.Error("create QR code failed,e=%v", err)
return nil, false
@@ -113,6 +119,37 @@ func callQRCodeCreate(sceneStr string) (*QRCodeResponse, bool) {
return &result, false
}

//getMaterial
// api doc: https://developers.weixin.qq.com/doc/offiaccount/Asset_Management/Get_materials_list.html
func getMaterial(mType string, offset, count int) (interface{}, bool) {
client := getWechatRestyClient()

body := &MaterialRequest{
Type: mType,
Offset: offset,
Count: count,
}
bodyJson, _ := json.Marshal(body)
r, err := client.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("access_token", GetWechatAccessToken()).
SetBody(bodyJson).
Post(setting.WechatApiHost + GET_MATERIAL_PATH)
if err != nil {
log.Error("create QR code failed,e=%v", err)
return nil, false
}
a := r.Body()
resultMap := make(map[string]interface{}, 0)
json.Unmarshal(a, &resultMap)
errcode := resultMap["errcode"]
if errcode == fmt.Sprint(ERR_CODE_ACCESSTOKEN_EXPIRE) || errcode == fmt.Sprint(ERR_CODE_ACCESSTOKEN_INVALID) {
return nil, true
}
log.Info("%v", r)
return &resultMap, false
}

func getErrorCodeFromResponse(r *resty.Response) int {
a := r.Body()
resultMap := make(map[string]interface{}, 0)


+ 97
- 4
modules/auth/wechat/event_handle.go View File

@@ -18,7 +18,7 @@ import (
// <EventKey><![CDATA[SCENE_VALUE]]></EventKey>
// <Ticket><![CDATA[TICKET]]></Ticket>
//</xml>
type WechatEvent struct {
type WechatMsg struct {
ToUserName string
FromUserName string
CreateTime int64
@@ -26,9 +26,13 @@ type WechatEvent struct {
Event string
EventKey string
Ticket string
Content string
MsgId string
MsgDataId string
Idx string
}

type EventReply struct {
type MsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
@@ -37,16 +41,97 @@ type EventReply struct {
Content string
}

type TextMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
Content string
}
type ImageMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
Image ImageContent
}
type VoiceMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
Voice VoiceContent
}
type VideoMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
Video VideoContent
}
type MusicMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
Music MusicContent
}
type NewsMsgReply struct {
XMLName xml.Name `xml:"xml"`
ToUserName string
FromUserName string
CreateTime int64
MsgType string
ArticleCount int
Articles ArticleItem
}

type ArticleItem struct {
Item []ArticlesContent
}

type ImageContent struct {
MediaId string
}
type VoiceContent struct {
MediaId string
}
type VideoContent struct {
MediaId string
Title string
Description string
}
type MusicContent struct {
Title string
Description string
MusicUrl string
HQMusicUrl string
ThumbMediaId string
}
type ArticlesContent struct {
XMLName xml.Name `xml:"item"`
Title string
Description string
PicUrl string
Url string
}

const (
WECHAT_EVENT_SUBSCRIBE = "subscribe"
WECHAT_EVENT_SCAN = "SCAN"
)

const (
WECHAT_MSG_TYPE_TEXT = "text"
WECHAT_MSG_TYPE_TEXT = "text"
WECHAT_MSG_TYPE_EVENT = "event"
)

func HandleSubscribeEvent(we WechatEvent) string {
func HandleScanEvent(we WechatMsg) string {
eventKey := we.EventKey
if eventKey == "" {
return ""
@@ -74,3 +159,11 @@ func HandleSubscribeEvent(we WechatEvent) string {

return BIND_REPLY_SUCCESS
}

func HandleSubscribeEvent(we WechatMsg) *WechatReplyContent {
r, err := LoadReplyFromCacheAndDisk(SubscribeReply)
if err != nil || len(r) == 0 {
return nil
}
return r[0]
}

+ 13
- 0
modules/auth/wechat/material.go View File

@@ -0,0 +1,13 @@
package wechat

import "code.gitea.io/gitea/modules/log"

func GetWechatMaterial(mType string, offset, count int) interface{} {
result, retryFlag := getMaterial(mType, offset, count)
if retryFlag {
log.Info("retryGetWechatMaterial calling")
refreshAccessToken()
result, _ = getMaterial(mType, offset, count)
}
return result
}

+ 184
- 144
modules/cloudbrain/cloudbrain.go View File

@@ -44,17 +44,40 @@ var (
TrainResourceSpecs *models.ResourceSpecs
)

type GenerateCloudBrainTaskReq struct {
Ctx *context.Context
DisplayJobName string
JobName string
Image string
Command string
CodePath string
ModelPath string
BenchmarkPath string
Snn4ImageNetPath string
BrainScorePath string
JobType string
GpuQueue string
Description string
BranchName string
BootFile string
Params string
CommitID string
Uuids string
DatasetNames string
DatasetInfos map[string]models.DatasetInfo
BenchmarkTypeID int
BenchmarkChildTypeID int
ResourceSpecId int
}

func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool {
if !ctx.IsSigned {
return false
}
log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner()))
log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin()))
if err != nil {

return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin()
} else {
log.Info("is job creator:" + strconv.FormatBool(ctx.User.ID == job.UserID))
return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID
}

@@ -187,23 +210,17 @@ func AdminOrImageCreaterRight(ctx *context.Context) {

}

func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description, branchName, bootFile, params, commitID string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error {

dataActualPath := setting.Attachment.Minio.RealPath +
setting.Attachment.Minio.Bucket + "/" +
setting.Attachment.Minio.BasePath +
models.AttachmentRelativePath(uuid) +
uuid

func GenerateTask(req GenerateCloudBrainTaskReq) error {
var resourceSpec *models.ResourceSpec
var versionCount int
if jobType == string(models.JobTypeTrain) {

if req.JobType == string(models.JobTypeTrain) {
versionCount = 1
if TrainResourceSpecs == nil {
json.Unmarshal([]byte(setting.TrainResourceSpecs), &TrainResourceSpecs)
}
for _, spec := range TrainResourceSpecs.ResourceSpec {
if resourceSpecId == spec.Id {
if req.ResourceSpecId == spec.Id {
resourceSpec = spec
}
}
@@ -212,7 +229,7 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command,
json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs)
}
for _, spec := range ResourceSpecs.ResourceSpec {
if resourceSpecId == spec.Id {
if req.ResourceSpecId == spec.Id {
resourceSpec = spec
}
}
@@ -220,25 +237,74 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command,
}

if resourceSpec == nil {
log.Error("no such resourceSpecId(%d)", resourceSpecId, ctx.Data["MsgID"])
log.Error("no such resourceSpecId(%d)", req.ResourceSpecId, req.Ctx.Data["MsgID"])
return errors.New("no such resourceSpec")
}

var datasetName string
attach, err := models.GetAttachmentByUUID(uuid)
if err != nil {
//for benchmark, do not return error
log.Error("GetAttachmentByUUID failed:%v", err)
volumes := []models.Volume{
{
HostPath: models.StHostPath{
Path: req.CodePath,
MountPath: CodeMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: req.ModelPath,
MountPath: ModelMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: req.BenchmarkPath,
MountPath: BenchMarkMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: req.Snn4ImageNetPath,
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: req.BrainScorePath,
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
}

if len(req.DatasetInfos) == 1 {
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: req.DatasetInfos[req.Uuids].DataLocalPath,
MountPath: DataSetMountPath,
ReadOnly: true,
},
})
} else {
datasetName = attach.Name
for _, dataset := range req.DatasetInfos {
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: dataset.DataLocalPath,
MountPath: DataSetMountPath + "/" + dataset.Name,
ReadOnly: true,
},
})
}
}

createTime := timeutil.TimeStampNow()
jobResult, err := CreateJob(jobName, models.CreateJobParams{
JobName: jobName,
jobResult, err := CreateJob(req.JobName, models.CreateJobParams{
JobName: req.JobName,
RetryCount: 1,
GpuType: gpuQueue,
Image: image,
GpuType: req.GpuQueue,
Image: req.Image,
TaskRoles: []models.TaskRole{
{
Name: SubTaskName,
@@ -249,94 +315,51 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command,
GPUNumber: resourceSpec.GpuNum,
MemoryMB: resourceSpec.MemMiB,
ShmMB: resourceSpec.ShareMemMiB,
Command: command,
Command: req.Command,
NeedIBDevice: false,
IsMainRole: false,
UseNNI: false,
},
},
Volumes: []models.Volume{
{
HostPath: models.StHostPath{
Path: codePath,
MountPath: CodeMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: dataActualPath,
MountPath: DataSetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: modelPath,
MountPath: ModelMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: benchmarkPath,
MountPath: BenchMarkMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: snn4imagenetPath,
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: brainScorePath,
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
},
Volumes: volumes,
})
if err != nil {
log.Error("CreateJob failed:", err.Error(), ctx.Data["MsgID"])
log.Error("CreateJob failed:", err.Error(), req.Ctx.Data["MsgID"])
return err
}
if jobResult.Code != Success {
log.Error("CreateJob(%s) failed:%s", jobName, jobResult.Msg, ctx.Data["MsgID"])
log.Error("CreateJob(%s) failed:%s", req.JobName, jobResult.Msg, req.Ctx.Data["MsgID"])
return errors.New(jobResult.Msg)
}

var jobID = jobResult.Payload["jobId"].(string)
err = models.CreateCloudbrain(&models.Cloudbrain{
Status: string(models.JobWaiting),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
UserID: req.Ctx.User.ID,
RepoID: req.Ctx.Repo.Repository.ID,
JobID: jobID,
JobName: jobName,
DisplayJobName: displayJobName,
JobName: req.JobName,
DisplayJobName: req.DisplayJobName,
SubTaskName: SubTaskName,
JobType: jobType,
JobType: req.JobType,
Type: models.TypeCloudBrainOne,
Uuid: uuid,
Image: image,
GpuQueue: gpuQueue,
ResourceSpecId: resourceSpecId,
Uuid: req.Uuids,
Image: req.Image,
GpuQueue: req.GpuQueue,
ResourceSpecId: req.ResourceSpecId,
ComputeResource: models.GPUResource,
BenchmarkTypeID: benchmarkTypeID,
BenchmarkChildTypeID: benchmarkChildTypeID,
Description: description,
BenchmarkTypeID: req.BenchmarkTypeID,
BenchmarkChildTypeID: req.BenchmarkChildTypeID,
Description: req.Description,
IsLatestVersion: "1",
VersionCount: versionCount,
BranchName: branchName,
BootFile: bootFile,
DatasetName: datasetName,
Parameters: params,
BranchName: req.BranchName,
BootFile: req.BootFile,
DatasetName: req.DatasetNames,
Parameters: req.Params,
CreatedUnix: createTime,
UpdatedUnix: createTime,
CommitID: commitID,
CommitID: req.CommitID,
})

if err != nil {
@@ -345,17 +368,17 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command,

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByName failed: %v", err.Error())
log.Error("GetCloudbrainByJobID failed: %v", err.Error())
return err
}
stringId := strconv.FormatInt(task.ID, 10)

if IsBenchmarkJob(jobType) {
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateBenchMarkTask)
} else if string(models.JobTypeTrain) == jobType {
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, displayJobName, models.ActionCreateGPUTrainTask)
if IsBenchmarkJob(req.JobType) {
notification.NotifyOtherTask(req.Ctx.User, req.Ctx.Repo.Repository, stringId, req.DisplayJobName, models.ActionCreateBenchMarkTask)
} else if string(models.JobTypeTrain) == req.JobType {
notification.NotifyOtherTask(req.Ctx.User, req.Ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateGPUTrainTask)
} else {
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugGPUTask)
notification.NotifyOtherTask(req.Ctx.User, req.Ctx.Repo.Repository, stringId, req.DisplayJobName, models.ActionCreateDebugGPUTask)
}

return nil
@@ -366,11 +389,6 @@ func IsBenchmarkJob(jobType string) bool {
}

func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) error {
dataActualPath := setting.Attachment.Minio.RealPath +
setting.Attachment.Minio.Bucket + "/" +
setting.Attachment.Minio.BasePath +
models.AttachmentRelativePath(task.Uuid) +
task.Uuid
jobName := task.JobName

var resourceSpec *models.ResourceSpec
@@ -388,6 +406,70 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e
return errors.New("no such resourceSpec")
}

datasetInfos, _, err := models.GetDatasetInfo(task.Uuid)
if err != nil {
log.Error("GetDatasetInfo failed:%v", err, ctx.Data["MsgID"])
return err
}

volumes := []models.Volume{
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, CodeMountPath+"/"),
MountPath: CodeMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, ModelMountPath+"/"),
MountPath: ModelMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, BenchMarkMountPath+"/"),
MountPath: BenchMarkMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath+"/"),
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, BrainScoreMountPath+"/"),
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
}

if len(datasetInfos) == 1 {
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: datasetInfos[task.Uuid].DataLocalPath,
MountPath: DataSetMountPath,
ReadOnly: true,
},
})
} else {
for _, dataset := range datasetInfos {
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: dataset.DataLocalPath,
MountPath: DataSetMountPath + "/" + dataset.Name,
ReadOnly: true,
},
})
}
}

createTime := timeutil.TimeStampNow()
jobResult, err := CreateJob(jobName, models.CreateJobParams{
JobName: jobName,
@@ -410,50 +492,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e
UseNNI: false,
},
},
Volumes: []models.Volume{
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, CodeMountPath+"/"),
MountPath: CodeMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: dataActualPath,
MountPath: DataSetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, ModelMountPath+"/"),
MountPath: ModelMountPath,
ReadOnly: false,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, BenchMarkMountPath+"/"),
MountPath: BenchMarkMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath+"/"),
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: storage.GetMinioPath(jobName, BrainScoreMountPath+"/"),
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
},
Volumes: volumes,
})
if err != nil {
log.Error("CreateJob failed:%v", err.Error(), ctx.Data["MsgID"])
@@ -476,6 +515,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e
JobType: task.JobType,
Type: task.Type,
Uuid: task.Uuid,
DatasetName: task.DatasetName,
Image: task.Image,
GpuQueue: task.GpuQueue,
ResourceSpecId: task.ResourceSpecId,


+ 217
- 0
modules/grampus/grampus.go View File

@@ -0,0 +1,217 @@
package grampus

import (
"encoding/json"
"strings"

"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/timeutil"
)

const (
JobPath = "job/"

ProcessorTypeNPU = "npu.huawei.com/NPU"
ProcessorTypeGPU = "nvidia.com/gpu"

GpuWorkDir = "/tmp/"
NpuWorkDir = "/cache/"

CommandPrepareScript = ";mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
"echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;"
//CommandPrepareScript = "pwd;cd /cache;mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
// "echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;"

CodeArchiveName = "master.zip"
)

var (
poolInfos *models.PoolInfos
FlavorInfos *models.FlavorInfos
ImageInfos *models.ImageInfosModelArts

SpecialPools *models.SpecialPools
)

type GenerateTrainJobReq struct {
JobName string
Command string
ResourceSpecId string
ImageUrl string //与image_id二选一,都有的情况下优先image_url
ImageId string

DisplayJobName string
Uuid string
Description string
CodeObsPath string
BootFile string
BootFileUrl string
DataUrl string
TrainUrl string
WorkServerNumber int
EngineID int64
CommitID string
IsLatestVersion string
BranchName string
PreVersionId int64
PreVersionName string
FlavorName string
VersionCount int
EngineName string
TotalVersionCount int
ComputeResource string
ProcessType string
DatasetName string
Params string
}

func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) {
createTime := timeutil.TimeStampNow()

centerID, centerName := getCentersParamter(ctx, req)

jobResult, err := createJob(models.CreateGrampusJobRequest{
Name: req.JobName,
Tasks: []models.GrampusTasks{
{
Name: req.JobName,
Command: req.Command,
ResourceSpecId: req.ResourceSpecId,
ImageId: req.ImageId,
ImageUrl: req.ImageUrl,
CenterID: centerID,
CenterName: centerName,
ReplicaNum: 1,
},
},
})
if err != nil {
log.Error("createJob failed: %v", err.Error())
return err
}

jobID := jobResult.JobInfo.JobID
err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.JobInfo.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobID,
JobName: req.JobName,
DisplayJobName: req.DisplayJobName,
JobType: string(models.JobTypeTrain),
Type: models.TypeC2Net,
Uuid: req.Uuid,
DatasetName: req.DatasetName,
CommitID: req.CommitID,
IsLatestVersion: req.IsLatestVersion,
ComputeResource: req.ComputeResource,
ImageID: req.ImageId,
TrainUrl: req.TrainUrl,
BranchName: req.BranchName,
Parameters: req.Params,
BootFile: req.BootFile,
DataUrl: req.DataUrl,
FlavorCode: req.ResourceSpecId,
Description: req.Description,
WorkServerNumber: req.WorkServerNumber,
FlavorName: req.FlavorName,
EngineName: req.EngineName,
VersionCount: req.VersionCount,
TotalVersionCount: req.TotalVersionCount,
CreatedUnix: createTime,
UpdatedUnix: createTime,
})

if err != nil {
log.Error("CreateCloudbrain(%s) failed:%v", req.DisplayJobName, err.Error())
return err
}

var actionType models.ActionType
if req.ComputeResource == models.NPUResource {
actionType = models.ActionCreateGrampusNPUTrainTask
} else if req.ComputeResource == models.GPUResource {
actionType = models.ActionCreateGrampusGPUTrainTask
}
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, actionType)

return nil
}

func getCentersParamter(ctx *context.Context, req *GenerateTrainJobReq) ([]string, []string) {
var centerID []string
var centerName []string

includeCenters := make(map[string]string)
excludeCenters := make(map[string]string)

if SpecialPools != nil {
for _, pool := range SpecialPools.Pools {
if !pool.IsExclusive && strings.Contains(req.ComputeResource, pool.Type) {
org, _ := models.GetOrgByName(pool.Org)
if org != nil {
isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID)
if isOrgMember {
for _, info := range pool.Pool {
includeCenters[info.Queue] = info.Value
}
} else {
for _, info := range pool.Pool {
excludeCenters[info.Queue] = info.Value
}
}
}
}
}

}

if len(includeCenters) > 0 {
//如果有专属资源池,根据专属资源池指定智算中心
for k, v := range includeCenters {
centerID = append(centerID, k)
centerName = append(centerName, v)
}
} else if len(excludeCenters) > 0 {
//否则,有要排除的中心,先获取所有中心,删除其中的排除中心,得到指定的智算中心
allCenters := make(map[string]string)
specs, err := GetResourceSpecs(req.ProcessType)
if err == nil {
for _, info := range specs.Infos {
for _, center := range info.Centers {
allCenters[center.ID] = center.Name
}

}
}

for k, _ := range excludeCenters {
delete(allCenters, k)
}

for k, v := range allCenters {
centerID = append(centerID, k)
centerName = append(centerName, v)
}

}
return centerID, centerName
}

func TransTrainJobStatus(status string) string {
if status == models.GrampusStatusPending {
status = models.GrampusStatusWaiting
}

return strings.ToUpper(status)
}
func InitSpecialPool() {
if SpecialPools == nil && setting.Grampus.SpecialPools != "" {
json.Unmarshal([]byte(setting.Grampus.SpecialPools), &SpecialPools)
}
}

+ 277
- 0
modules/grampus/resty.go View File

@@ -0,0 +1,277 @@
package grampus

import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"crypto/tls"
"encoding/json"
"fmt"
"github.com/go-resty/resty/v2"
"net/http"
)

var (
restyClient *resty.Client
HOST string
TOKEN string
)

const (
urlOpenApiV1 = "/openapi/v1/"

urlGetToken = urlOpenApiV1 + "token"
urlTrainJob = urlOpenApiV1 + "trainjob"
urlGetResourceSpecs = urlOpenApiV1 + "resourcespec"
urlGetImages = urlOpenApiV1 + "image"

errorIllegalToken = 1005
)

type GetTokenParams struct {
UserName string `json:"username"`
Password string `json:"password"`
}

type GetTokenResult struct {
Token string `json:"token"`
Expiration int64 `json:"expiration"`
}

func getRestyClient() *resty.Client {
if restyClient == nil {
restyClient = resty.New()
restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
return restyClient
}

func checkSetting() {
if len(HOST) != 0 && len(TOKEN) != 0 && restyClient != nil {
return
}

err := getToken()
if err != nil {
log.Error("getToken failed:%v", err)
}
}

func getToken() error {
HOST = setting.Grampus.Host

client := getRestyClient()
params := GetTokenParams{
UserName: setting.Grampus.UserName,
Password: setting.Grampus.Password,
}

var result GetTokenResult
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetBody(params).
SetResult(&result).
Post(HOST + urlGetToken)
if err != nil {
return fmt.Errorf("resty getToken: %v", err)
}

if res.StatusCode() != http.StatusOK {
return fmt.Errorf("getToken failed:%s", res.String())
}

TOKEN = result.Token

return nil
}

func createJob(req models.CreateGrampusJobRequest) (*models.CreateGrampusJobResponse, error) {
checkSetting()
client := getRestyClient()
var result models.CreateGrampusJobResponse

retry := 0

sendjob:
_, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(req).
SetResult(&result).
Post(HOST + urlTrainJob)

if err != nil {
return nil, fmt.Errorf("resty CreateJob: %s", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("CreateJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("CreateJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetJob(jobID string) (*models.GetGrampusJobResponse, error) {
checkSetting()
client := getRestyClient()
var result models.GetGrampusJobResponse

retry := 0

sendjob:
_, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + urlTrainJob + "/" + jobID)

if err != nil {
return nil, fmt.Errorf("resty GetJob: %v", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return nil, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetResourceSpecs(processorType string) (*models.GetGrampusResourceSpecsResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetGrampusResourceSpecsResult

retry := 0

sendjob:
_, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + urlGetResourceSpecs + "?processorType=" + processorType)

if err != nil {
return nil, fmt.Errorf("resty GetResourceSpecs: %v", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("GetResourceSpecs failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("GetResourceSpecs failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetImages(processorType string) (*models.GetGrampusImagesResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetGrampusImagesResult

retry := 0

sendjob:
_, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + urlGetImages + "?processorType=" + processorType)

if err != nil {
return nil, fmt.Errorf("resty GetImages: %v", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("GetImages failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("GetImages failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetTrainJobLog(jobID string) (string, error) {
checkSetting()
client := getRestyClient()
var logContent string

res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&logContent).
Get(HOST + urlTrainJob + "/" + jobID + "/task/0/replica/0/log")

if err != nil {
return logContent, fmt.Errorf("resty GetTrainJobLog: %v", err)
}

if res.StatusCode() != http.StatusOK {
var temp models.GrampusResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return logContent, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJobLog failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return logContent, fmt.Errorf("GetTrainJobLog failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

logContent = res.String()

return logContent, nil
}

func StopJob(jobID string) (*models.GrampusStopJobResponse, error) {
checkSetting()
client := getRestyClient()
var result models.GrampusStopJobResponse

retry := 0

sendjob:
_, err := client.R().
//SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetResult(&result).
Post(HOST + urlTrainJob + "/" + jobID + "/stop")

if err != nil {
return &result, fmt.Errorf("resty StopTrainJob: %v", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

+ 6
- 14
modules/modelarts/modelarts.go View File

@@ -1,13 +1,14 @@
package modelarts

import (
"code.gitea.io/gitea/modules/timeutil"
"encoding/json"
"errors"
"fmt"
"path"
"strconv"

"code.gitea.io/gitea/modules/timeutil"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
@@ -50,6 +51,7 @@ const (
Lines = 500
TrainUrl = "train_url"
DataUrl = "data_url"
MultiDataUrl = "multi_data_url"
ResultUrl = "result_url"
CkptUrl = "ckpt_url"
DeviceTarget = "device_target"
@@ -96,6 +98,7 @@ type GenerateTrainJobReq struct {
VersionCount int
EngineName string
TotalVersionCount int
DatasetName string
}

type GenerateInferenceJobReq struct {
@@ -335,11 +338,6 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error
return err
}

attach, err := models.GetAttachmentByUUID(req.Uuid)
if err != nil {
log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error())
return err
}
jobId := strconv.FormatInt(jobResult.JobID, 10)
err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.Status),
@@ -353,7 +351,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
DatasetName: req.DatasetName,
CommitID: req.CommitID,
IsLatestVersion: req.IsLatestVersion,
ComputeResource: models.NPUResource,
@@ -408,12 +406,6 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
return err
}

attach, err := models.GetAttachmentByUUID(req.Uuid)
if err != nil {
log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error())
return err
}

var jobTypes []string
jobTypes = append(jobTypes, string(models.JobTypeTrain))
repo := ctx.Repo.Repository
@@ -441,7 +433,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
DatasetName: req.DatasetName,
CommitID: req.CommitID,
IsLatestVersion: req.IsLatestVersion,
PreVersionName: req.PreVersionName,


+ 35
- 0
modules/setting/setting.go View File

@@ -465,6 +465,7 @@ var (
MaxDuration int64
TrainGpuTypes string
TrainResourceSpecs string
MaxDatasetNum int

//benchmark config
IsBenchmarkEnabled bool
@@ -528,6 +529,15 @@ var (
FlavorInfos string
TrainJobFLAVORINFOS string

//grampus config
Grampus = struct {
Env string
Host string
UserName string
Password string
SpecialPools string
}{}

//elk config
ElkUrl string
ElkUser string
@@ -545,6 +555,13 @@ var (
WechatQRCodeExpireSeconds int
WechatAuthSwitch bool

//wechat auto reply config
UserNameOfWechatReply string
RepoNameOfWechatReply string
RefNameOfWechatReply string
TreePathOfAutoMsgReply string
TreePathOfSubscribe string

//nginx proxy
PROXYURL string
RadarMap = struct {
@@ -1294,6 +1311,7 @@ func NewContext() {
MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400)
TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("")
TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("")
MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5)

sec = Cfg.Section("benchmark")
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
@@ -1372,6 +1390,11 @@ func NewContext() {
WechatAppSecret = sec.Key("APP_SECRET").MustString("e48e13f315adc32749ddc7057585f198")
WechatQRCodeExpireSeconds = sec.Key("QR_CODE_EXPIRE_SECONDS").MustInt(120)
WechatAuthSwitch = sec.Key("AUTH_SWITCH").MustBool(true)
UserNameOfWechatReply = sec.Key("AUTO_REPLY_USER_NAME").MustString("OpenIOSSG")
RepoNameOfWechatReply = sec.Key("AUTO_REPLY_REPO_NAME").MustString("promote")
RefNameOfWechatReply = sec.Key("AUTO_REPLY_REF_NAME").MustString("master")
TreePathOfAutoMsgReply = sec.Key("AUTO_REPLY_TREE_PATH").MustString("wechat/auto_reply.json")
TreePathOfSubscribe = sec.Key("SUBSCRIBE_TREE_PATH").MustString("wechat/subscribe_reply.json")

SetRadarMapConfig()

@@ -1382,6 +1405,18 @@ func NewContext() {
Course.OrgName = sec.Key("org_name").MustString("")
Course.TeamName = sec.Key("team_name").MustString("")

GetGrampusConfig()
}

func GetGrampusConfig() {
sec := Cfg.Section("grampus")

Grampus.Env = sec.Key("ENV").MustString("TEST")
Grampus.Host = sec.Key("SERVER_HOST").MustString("")
Grampus.UserName = sec.Key("USERNAME").MustString("")
Grampus.Password = sec.Key("PASSWORD").MustString("")
Grampus.SpecialPools = sec.Key("SPECIAL_POOL").MustString("")

}

func SetRadarMapConfig() {


+ 17
- 8
modules/setting/webhook.go View File

@@ -6,6 +6,7 @@ package setting

import (
"net/url"
"strings"

"code.gitea.io/gitea/modules/log"
)
@@ -13,14 +14,18 @@ import (
var (
// Webhook settings
Webhook = struct {
QueueLength int
DeliverTimeout int
SkipTLSVerify bool
Types []string
PagingNum int
ProxyURL string
ProxyURLFixed *url.URL
ProxyHosts []string
QueueLength int
DeliverTimeout int
SkipTLSVerify bool
Types []string
PagingNum int
ProxyURL string
ProxyURLFixed *url.URL
ProxyHosts []string
Socks5Proxy string
Socks5UserName string
Socks5Password string
Socks5ProxyHosts []string
}{
QueueLength: 1000,
DeliverTimeout: 5,
@@ -39,6 +44,10 @@ func newWebhookService() {
Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix"}
Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10)
Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("")
Webhook.Socks5Proxy = sec.Key("SOCKS5_PROXY_URL").MustString("")
Webhook.Socks5UserName = sec.Key("SOCKS5_USER_NAME").MustString("")
Webhook.Socks5Password = sec.Key("SOCKS5_PASSWORD").MustString("")
Webhook.Socks5ProxyHosts = strings.Split(sec.Key("SOCKS5_PROXY_HOST").MustString(""), ";")
if Webhook.ProxyURL != "" {
var err error
Webhook.ProxyURLFixed, err = url.Parse(Webhook.ProxyURL)


+ 10
- 0
modules/util/path.go View File

@@ -31,3 +31,13 @@ func GetDirectorySize(path string) (int64, error) {
})
return size, err
}

// check whether the path is dir
func IsDir(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}

return s.IsDir()
}

+ 1
- 1
modules/util/util.go View File

@@ -115,7 +115,7 @@ func AddZero(t int64) (m string) {

func ConvertDisplayJobNameToJobName(DisplayName string) (JobName string) {
t := time.Now()
JobName = "openi" + strings.ToLower(cutNameString(DisplayName, 15)) + "t" + t.Format("20060102150405")[4:] + strconv.Itoa(int(rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(100000)))
JobName = strings.ToLower(cutNameString(DisplayName, 15)) + "t" + t.Format("20060102150405")[10:] + strconv.Itoa(int(rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(100000)))
return JobName
}



+ 53
- 4
modules/webhook/deliver.go View File

@@ -16,6 +16,8 @@ import (
"sync"
"time"

"golang.org/x/net/proxy"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
@@ -137,8 +139,10 @@ func Deliver(t *models.HookTask) error {
return
}
}()
match := isSocks5ProxyUrlMatch(req)

resp, err := makeReq(req, match)

resp, err := webhookHTTPClient.Do(req)
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
return err
@@ -161,6 +165,23 @@ func Deliver(t *models.HookTask) error {
return nil
}

func makeReq(req *http.Request, proxyMatch bool) (*http.Response, error) {
if proxyMatch {
return webhookSocks5PoxyHTTPClient.Do(req)
}
return webhookHTTPClient.Do(req)
}

func isSocks5ProxyUrlMatch(req *http.Request) bool {

for _, v := range socks5HostMatchers {
if v.Match(req.URL.Host) {
return true
}
}
return false
}

// DeliverHooks checks and delivers undelivered hooks.
// FIXME: graceful: This would likely benefit from either a worker pool with dummy queue
// or a full queue. Then more hooks could be sent at same time.
@@ -225,9 +246,11 @@ func DeliverHooks(ctx context.Context) {
}

var (
webhookHTTPClient *http.Client
once sync.Once
hostMatchers []glob.Glob
webhookHTTPClient *http.Client
once sync.Once
hostMatchers []glob.Glob
webhookSocks5PoxyHTTPClient *http.Client
socks5HostMatchers []glob.Glob
)

func webhookProxy() func(req *http.Request) (*url.URL, error) {
@@ -274,5 +297,31 @@ func InitDeliverHooks() {
},
}

if setting.Webhook.Socks5Proxy != "" {
auth := proxy.Auth{
User: setting.Webhook.Socks5UserName,
Password: setting.Webhook.Socks5Password,
}

dialSocksProxy, err := proxy.SOCKS5("tcp", setting.Webhook.Socks5Proxy, &auth, proxy.Direct)
if err != nil {
fmt.Println("Error connecting to proxy:", err)
}
tr := &http.Transport{Dial: dialSocksProxy.Dial}

webhookSocks5PoxyHTTPClient = &http.Client{
Transport: tr,
}

for _, h := range setting.Webhook.Socks5ProxyHosts {
if g, err := glob.Compile(h); err == nil {
socks5HostMatchers = append(socks5HostMatchers, g)
} else {
log.Error("glob.Compile %s failed: %v", h, err)
}
}

}

go graceful.GetManager().RunWithShutdownContext(DeliverHooks)
}

+ 14
- 0
options/locale/locale_en-US.ini View File

@@ -940,6 +940,7 @@ unzip_successed=Unzip Successed
unzip_failed=Unzip Failed
unzip_stared=Unzipping
unzip_status=Unzip Status
collection_num=Collection Nums
[repo]
owner = Owner
repo_name = Repository Name
@@ -1034,6 +1035,7 @@ image_delete_fail=Failed to delete image, please try again later.
image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image?
download=Download
score=Score
file_limit_100 = Display up to 100 files or folders in a single directory

images.name = Image Tag
images.name_placerholder = Please enter the image name
@@ -1214,6 +1216,11 @@ model.manage.sava_model = Sava Model
model.manage.model_manage = ModelManage
model.manage.model_accuracy = Model Accuracy

grampus.train_job.ai_center = AI Center
grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。
grampus.gpu_dataset_path_rule = The code is storaged in /tmp/code;the dataset is storaged in /tmp/dataset;and please put your model into /tmp/output, then you can download it online。
grampus.no_operate_right = You have no right to do this operation.

template.items = Template Items
template.git_content = Git Content (Default Branch)
template.git_hooks = Git Hooks
@@ -2974,6 +2981,8 @@ task_inferencejob=`created reasoning task <a href="%s/modelarts/inference-job/%s
task_benchmark=`created profiling task <a href="%s/cloudbrain/benchmark/%s">%s</a>`
task_createmodel=`created new model <a href="%s/modelmanage/show_model_info?name=%s">%s</a>`
task_gputrainjob=`created CPU/GPU training task<a href="%s/cloudbrain/train-job/%s">%s</a>`
task_c2netnputrainjob=`created NPU training task<a href="%s/grampus/train-job/%s">%s</a>`
task_c2netgputrainjob=`created CPU/GPU training task<a href="%s/grampus/train-job/%s">%s</a>`

[tool]
ago = %s ago
@@ -3062,6 +3071,9 @@ Platform_Tutorial = Tutorial
foot.advice_feedback = Feedback

[cloudbrain]
resource_cluster = Resource Cluster
resource_cluster_openi = OpenI Resource Cluster
resource_cluster_c2net = China Computing NET
compute_resource = Computing resources
task_name = Task name
task_type = Task type
@@ -3107,3 +3119,5 @@ TRAIN = TRAIN
INFERENCE = INFERENCE
BENCHMARK = BENCHMARK
brain_area = Brain Area

error.dataset_select = dataset select error:the count exceed the limit or has same name

+ 16
- 4
options/locale/locale_zh-CN.ini View File

@@ -945,7 +945,7 @@ unzip_successed=解压成功
unzip_failed=解压失败
unzip_stared=解压中
unzip_status=解压状态
collection_num=收藏数量
[repo]
owner=拥有者
repo_name=项目名称
@@ -1035,7 +1035,7 @@ image_delete_fail=删除镜像失败,请稍后再试。
image_overwrite=您已经提交过相同名称的镜像,您确定要覆盖原来提交的镜像吗?
download=模型下载
score=评分
file_limit_100 = 单目录下最多显示100个文件或文件夹

images.name = 镜像Tag
images.name_placerholder = 请输入镜像Tag
@@ -1228,6 +1228,12 @@ model.manage.sava_model = 保存模型
model.manage.model_manage = 模型管理
model.manage.model_accuracy = 模型精度

grampus.train_job.ai_center=智算中心
grampus.dataset_path_rule = 训练脚本存储在/cache/code中,数据集存储在/cache/dataset中,训练输出请存储在/cache/output中以供后续下载。
grampus.gpu_dataset_path_rule = 训练脚本存储在/tmp/code中,数据集存储在/tmp/dataset中,训练输出请存储在/tmp/output中以供后续下载。

grampus.no_operate_right = 您没有权限创建这类任务。

template.items=模板选项
template.git_content=Git数据(默认分支)
template.git_hooks=Git 钩子
@@ -1458,7 +1464,7 @@ issues.label_templates.helper=选择标签模板
issues.label_templates.use=使用标签集
issues.label_templates.fail_to_load_file=加载标签模板文件 '%s' 时发生错误:%v
issues.add_label_at=添加了标签 <div class="ui label" style="color: %s\; background-color: %s"> %s </div> %s
issues.remove_label_at=删除了 <div class="ui label" style="color: %s\; background-color: %s">%s</div> label %s 标签
issues.remove_label_at=删除了标签 <div class="ui label" style="color: %s\; background-color: %s">%s </div> %s
issues.add_milestone_at=` %[2]s 添加了里程碑 <b>%[1]s</b>`
issues.change_milestone_at=`%[3]s 修改了里程碑从 <b>%[1]s</b> 到 <b>%[2]s</b>`
issues.remove_milestone_at=`%[2]s 删除了里程碑 <b>%[1]s</b>`
@@ -1506,7 +1512,7 @@ issues.filter_sort.mostforks=派生由多到少
issues.filter_sort.fewestforks=派生由少到多
issues.filter_sort.downloadtimes=下载次数
issues.filter_sort.citations=引用次数
issues.filter_sort.moststars=收藏数量
issues.filter_sort.moststars=点赞由多到少
issues.filter_sort.mostusecount=最多引用
issues.filter_sort.fewestusecount=最少引用

@@ -2989,6 +2995,8 @@ task_inferencejob=`创建了推理任务 <a href="%s/modelarts/inference-job/%s"
task_benchmark=`创建了评测任务 <a href="%s/cloudbrain/benchmark/%s">%s</a>`
task_createmodel=`导入了新模型 <a href="%s/modelmanage/show_model_info?name=%s">%s</a>`
task_gputrainjob=`创建了CPU/GPU类型训练任务 <a href="%s/cloudbrain/train-job/%s">%s</a>`
task_c2netnputrainjob=`创建了NPU类型训练任务 <a href="%s/grampus/train-job/%s">%s</a>`
task_c2netgputrainjob=`创建了CPU/GPU类型训练任务 <a href="%s/grampus/train-job/%s">%s</a>`

[tool]
ago=%s前
@@ -3077,6 +3085,9 @@ Platform_Tutorial=新手指引
foot.advice_feedback = 意见反馈

[cloudbrain]
resource_cluster = 算力集群
resource_cluster_openi = 启智集群
resource_cluster_c2net = 智算网络集群
compute_resource = 计算资源
task_name = 任务名称
task_type = 任务类型
@@ -3123,3 +3134,4 @@ INFERENCE = 推理任务
BENCHMARK = 评测任务
brain_area = 脑区

error.dataset_select = 数据集选择错误:数量超过限制或者有同名数据集

+ 31
- 18
public/home/home.js View File

@@ -74,28 +74,30 @@ var swiperOrg = new Swiper(".homeorg-list", {
},
});

var output = document.getElementById("newmessage");
var url = "ws://" + document.location.host + "/action/notification";
if(document.location.host == "git.openi.org.cn" || document.URL.startsWith("https")){
url = "wss://" + document.location.host + "/action/notification"
}
var socket = new WebSocket(url);

socket.onopen = function () {
messageQueue = [];
console.log("message has connected.");
};

var maxSize = 20;
var html =document.documentElement;
var lang = html.attributes["lang"]
var isZh = true;
if(lang != null && lang.nodeValue =="en-US" ){
isZh=false;
}else{
}

socket.onmessage = function (e) {
document.onreadystatechange = function () {
queryRecommendData();

var output = document.getElementById("newmessage");
var url = "ws://" + document.location.host + "/action/notification";
if(document.location.host == "git.openi.org.cn" || document.URL.startsWith("https")){
url = "wss://" + document.location.host + "/action/notification"
}
var socket = new WebSocket(url);
socket.onopen = function () {
messageQueue = [];
console.log("message has connected.");
};

socket.onmessage = function (e) {
var data =JSON.parse(e.data)
var html = "";
if (data != null){
@@ -154,7 +156,7 @@ socket.onmessage = function (e) {
html += recordPrefix + actionName;
html += " <a href=\"" + getRepoLink(record) + "\" rel=\"nofollow\">" + getRepotext(record) + "</a>"
}
else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31"){
else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){
html += recordPrefix + actionName;
html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
}
@@ -176,7 +178,10 @@ socket.onmessage = function (e) {
output.innerHTML = html;
swiperNewMessage.updateSlides();
swiperNewMessage.updateProgress();
};
};
}



function getTaskLink(record){
var re = getRepoLink(record);
@@ -196,6 +201,8 @@ function getTaskLink(record){
re = re + "/modelmanage/show_model_info?name=" + record.RefName;
}else if(record.OpType == 31){
re = re + "/cloudbrain/train-job/" + record.Content;
}else if(record.OpType == 32 || record.OpType == 33){
re = re + "/grampus/train-job/" + record.Content;
}
re = encodeURI(re);
return re;
@@ -369,7 +376,9 @@ var actionNameZH={
"28":"创建了推理任务",
"29":"创建了评测任务",
"30":"导入了新模型",
"31":"创建了CPU/GPU类型训练任务"
"31":"创建了CPU/GPU类型训练任务",
"32":"创建了NPU类型训练任务",
"33":"创建了CPU/GPU类型训练任务"
};

var actionNameEN={
@@ -396,6 +405,8 @@ var actionNameEN={
"29":" created profiling task",
"30":" created new model",
"31":" created CPU/GPU type training task",
"32":" created NPU type training task",
"33":" created CPU/GPU type training task"
};

var repoAndOrgZH={
@@ -437,7 +448,9 @@ function getAction(opType,isZh){
}
}

queryRecommendData();




function queryRecommendData(){
$.ajax({


+ 5
- 3
public/home/search.js View File

@@ -918,6 +918,8 @@ function LetterAvatar(name, size, color) {
} else {
initials = nameSplit[0].charAt(0) + nameSplit[1].charAt(0);
}
let initials1 = initials.toUpperCase();
initials.toUpperCase();
if (w.devicePixelRatio) {
size = size * w.devicePixelRatio;
}
@@ -934,7 +936,7 @@ function LetterAvatar(name, size, color) {
context.font = Math.round(canvas.width / 2) + "px 'Microsoft Yahei'";
context.textAlign = "center";
context.fillStyle = "#FFF";
context.fillText(initials, size / 2, size / 1.5);
context.fillText(initials1, size / 2, size / 1.5);
dataURI = canvas.toDataURL();
canvas = null;
return dataURI;
@@ -1009,9 +1011,9 @@ function displayRepoResult(page, jsonResult, onlyReturnNum, keyword) {
var recordMap = data[i];
html += '<div class="item">';
if (recordMap["avatar"]) {
html += `<img class="ui avatar image" style="width:28px;height:28px" src="${recordMap["avatar"]}">`;
html += `<img class="ui avatar image" src="${recordMap["avatar"]}">`;
} else {
html += `<img class="ui avatar image" style="width:28px;height:28px" avatar="${recordMap["owner_name"]}">`;
html += `<img class="ui avatar image" avatar="${recordMap["owner_name"]}">`;
}

html += ' <div class="content">';


+ 2
- 7
routers/admin/cloudbrains.go View File

@@ -43,12 +43,6 @@ func CloudBrains(ctx *context.Context) {
if page <= 0 {
page = 1
}
debugType := models.TypeCloudBrainAll
if listType == models.GPUResource {
debugType = models.TypeCloudBrainOne
} else if listType == models.NPUResource {
debugType = models.TypeCloudBrainTwo
}

var jobTypes []string
jobTypeNot := false
@@ -77,13 +71,14 @@ func CloudBrains(ctx *context.Context) {
PageSize: setting.UI.IssuePagingNum,
},
Keyword: keyword,
Type: debugType,
JobTypeNot: jobTypeNot,
JobStatusNot: jobStatusNot,
JobStatus: jobStatuses,
JobTypes: jobTypes,
NeedRepoInfo: true,
IsLatestVersion: modelarts.IsLatestVersion,
ComputeResource: listType,
Type: models.TypeCloudBrainAll,
})
if err != nil {
ctx.ServerError("Get job failed:", err)


+ 4
- 3
routers/admin/dataset.go View File

@@ -77,9 +77,10 @@ func Datasets(ctx *context.Context) {
Page: page,
PageSize: setting.UI.ExplorePagingNum,
},
Keyword: keyword,
RecommendOnly: ctx.QueryBool("recommend"),
SearchOrderBy: orderBy,
Keyword: keyword,
RecommendOnly: ctx.QueryBool("recommend"),
CloudBrainType: -1,
SearchOrderBy: orderBy,
})
if err != nil {
ctx.ServerError("SearchDataset", err)


+ 10
- 0
routers/api/v1/api.go View File

@@ -947,6 +947,15 @@ func RegisterRoutes(m *macaron.Macaron) {
})
})
}, reqRepoReader(models.UnitTypeCloudBrain))
m.Group("/grampus", func() {
m.Group("/train-job", func() {
m.Group("/:jobid", func() {
m.Get("", repo.GetModelArtsTrainJobVersion)
m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.GrampusStopJob)
m.Get("/log", repo_ext.GrampusGetLog)
})
})
}, reqRepoReader(models.UnitTypeCloudBrain))
}, repoAssignment())
})

@@ -1046,6 +1055,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/prd/event", authentication.ValidEventSource)
m.Post("/prd/event", authentication.AcceptWechatEvent)
})
m.Get("/wechat/material", authentication.GetMaterial)
}, securityHeaders(), context.APIContexter(), sudo())
}



+ 3
- 3
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -103,7 +103,7 @@ func GetAllCloudbrainsOverview(ctx *context.Context) {
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration
}
if cloudbrain.Cloudbrain.Type == models.TypeIntelligentNet {
if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration
}

@@ -540,7 +540,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
cloudTwoJobTypeRes[cloudbrain.JobType] += 1
}
}
if cloudbrain.Cloudbrain.Type == models.TypeIntelligentNet {
if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
if _, ok := intelligentNetJobTypeRes[cloudbrain.JobType]; !ok {
intelligentNetJobTypeRes[cloudbrain.JobType] = 1
} else {
@@ -1287,7 +1287,7 @@ func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string {
return ctx.Tr("repo.cloudbrain1")
} else if rs.Cloudbrain.Type == models.TypeCloudBrainTwo {
return ctx.Tr("repo.cloudbrain2")
} else if rs.Cloudbrain.Type == models.TypeIntelligentNet {
} else if rs.Cloudbrain.Type == models.TypeC2Net {
return ctx.Tr("repo.intelligent_net")
} else {
return ctx.Tr("repo.cloudbrain_untype")


+ 67
- 8
routers/api/v1/repo/modelarts.go View File

@@ -6,6 +6,8 @@
package repo

import (
"code.gitea.io/gitea/modules/grampus"
"encoding/json"
"net/http"
"strconv"
"strings"
@@ -125,7 +127,8 @@ func GetModelArtsTrainJob(ctx *context.APIContext) {

func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
var (
err error
err error
aiCenterName string
)

jobID := ctx.Params(":jobid")
@@ -167,7 +170,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
log.Error("UpdateJob failed:", err)
}
}
} else {
} else if job.Type == models.TypeCloudBrainTwo {
result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10))
if err != nil {
ctx.NotFound(err)
@@ -189,12 +192,50 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
if err != nil {
log.Error("UpdateJob failed:", err)
}
} else if job.Type == models.TypeC2Net {
result, err := grampus.GetJob(jobID)
if err != nil {
log.Error("GetJob(%s) failed:%v", job.JobName, err)
ctx.NotFound(err)
return
}

if job.StartTime == 0 && result.JobInfo.StartedAt > 0 {
job.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
job.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
job.Duration = result.JobInfo.RunSec
job.TrainJobDuration = models.ConvertDurationToStr(job.Duration)

if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
job.EndTime = job.StartTime.Add(job.Duration)
}
job.CorrectCreateUnix()

if len(job.AiCenter) == 0 {
if len(result.JobInfo.Tasks) > 0 {
if len(result.JobInfo.Tasks[0].CenterID) > 0 && len(result.JobInfo.Tasks[0].CenterName) > 0 {
job.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
aiCenterName = result.JobInfo.Tasks[0].CenterName[0]
}
}
} else {
temp := strings.Split(job.AiCenter, "+")
if len(temp) > 1 {
aiCenterName = temp[1]
}
}
err = models.UpdateTrainJobVersion(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobID": jobID,
"JobStatus": job.Status,
"JobDuration": job.TrainJobDuration,
"AiCenter": aiCenterName,
})

}
@@ -373,11 +414,29 @@ func ModelList(ctx *context.APIContext) {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
return
}
models, err := storage.GetObsListObject(task.JobName, "output/", parentDir, versionName)
if err != nil {
log.Info("get TrainJobListModel failed:", err)
ctx.ServerError("GetObsListObject:", err)
return

var fileInfos []storage.FileInfo
if task.ComputeResource == models.NPUResource {
fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName)
if err != nil {
log.Info("get TrainJobListModel failed:", err)
ctx.ServerError("GetObsListObject:", err)
return
}
} else if task.ComputeResource == models.GPUResource {
files, err := routerRepo.GetModelDirs(task.JobName, parentDir)
if err != nil {
log.Info("GetModelDirs failed:", err)
ctx.ServerError("GetModelDirs:", err)
return
}

err = json.Unmarshal([]byte(files), &fileInfos)
if err != nil {
log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("json.Unmarshal failed:", err)
return
}
}

ctx.JSON(http.StatusOK, map[string]interface{}{
@@ -385,7 +444,7 @@ func ModelList(ctx *context.APIContext) {
"VersionName": versionName,
"StatusOK": 0,
"Path": dirArray,
"Dirs": models,
"Dirs": fileInfos,
"task": task,
"PageIsCloudBrain": true,
})


+ 3
- 10
routers/api/v1/repo/repo_dashbord.go View File

@@ -887,19 +887,12 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time,
if queryType == "all" {
beginTime = recordBeginTimeTemp
endTime = now
} else if queryType == "today" {
} else if queryType == "yesterday" {
endTime = now
beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())
} else if queryType == "yesterday" {
endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
beginTime = endTime.AddDate(0, 0, -1)

} else if queryType == "last_7day" {
beginTime = now.AddDate(0, 0, -7)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "last_30day" {
beginTime = now.AddDate(0, 0, -30)
} else if queryType == "current_week" {
beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "current_month" {


+ 22
- 0
routers/authentication/wechat.go View File

@@ -8,9 +8,11 @@ import (
"code.gitea.io/gitea/modules/redis/redis_client"
"code.gitea.io/gitea/modules/redis/redis_key"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/routers/response"
"encoding/json"
"errors"
gouuid "github.com/satori/go.uuid"
"strconv"
"time"
)

@@ -124,3 +126,23 @@ func createQRCode4Bind(userId int64) (*QRCodeResponse, error) {
}
return result, nil
}

// GetMaterial
func GetMaterial(ctx *context.Context) {
mType := ctx.Query("type")
offsetStr := ctx.Query("offset")
countStr := ctx.Query("count")
var offset, count int
if offsetStr == "" {
offset = 0
} else {
offset, _ = strconv.Atoi(offsetStr)
}
if countStr == "" {
count = 20
} else {
count, _ = strconv.Atoi(countStr)
}
r := wechat.GetWechatMaterial(mType, offset, count)
ctx.JSON(200, response.SuccessWithData(r))
}

+ 129
- 15
routers/authentication/wechat_event.go View File

@@ -14,24 +14,48 @@ import (
// https://developers.weixin.qq.com/doc/offiaccount/Message_Management/Passive_user_reply_message.html
func AcceptWechatEvent(ctx *context.Context) {
b, _ := ioutil.ReadAll(ctx.Req.Request.Body)
we := wechat.WechatEvent{}
we := wechat.WechatMsg{}
xml.Unmarshal(b, &we)

switch we.MsgType {
case wechat.WECHAT_MSG_TYPE_EVENT:
HandleEventMsg(ctx, we)
case wechat.WECHAT_MSG_TYPE_TEXT:
HandleTextMsg(ctx, we)
}
log.Info("accept wechat event= %+v", we)
var replyStr string
switch we.Event {
case wechat.WECHAT_EVENT_SUBSCRIBE, wechat.WECHAT_EVENT_SCAN:
replyStr = wechat.HandleSubscribeEvent(we)
break

}

// ValidEventSource
func ValidEventSource(ctx *context.Context) {
echostr := ctx.Query("echostr")
ctx.Write([]byte(echostr))
return
}

func HandleEventMsg(ctx *context.Context, msg wechat.WechatMsg) {
switch msg.Event {
case wechat.WECHAT_EVENT_SCAN:
HandleEventScan(ctx, msg)
case wechat.WECHAT_EVENT_SUBSCRIBE:
if msg.EventKey != "" {
HandleEventScan(ctx, msg)
} else {
HandleEventSubscribe(ctx, msg)
}

}
}

func HandleEventScan(ctx *context.Context, msg wechat.WechatMsg) {
replyStr := wechat.HandleScanEvent(msg)
if replyStr == "" {
log.Info("reply str is empty")
return
}
reply := &wechat.EventReply{
ToUserName: we.FromUserName,
FromUserName: we.ToUserName,
reply := &wechat.MsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: wechat.WECHAT_MSG_TYPE_TEXT,
Content: replyStr,
@@ -39,9 +63,99 @@ func AcceptWechatEvent(ctx *context.Context) {
ctx.XML(200, reply)
}

// ValidEventSource
func ValidEventSource(ctx *context.Context) {
echostr := ctx.Query("echostr")
ctx.Write([]byte(echostr))
return
func HandleEventSubscribe(ctx *context.Context, msg wechat.WechatMsg) {
r := wechat.HandleSubscribeEvent(msg)
if r == nil {
return
}
reply := buildReplyContent(msg, r)
ctx.XML(200, reply)
}

func HandleTextMsg(ctx *context.Context, msg wechat.WechatMsg) {
r := wechat.GetAutomaticReply(msg.Content)
if r == nil {
log.Info("TextMsg reply is empty")
return
}
reply := buildReplyContent(msg, r)
ctx.XML(200, reply)
}

func buildReplyContent(msg wechat.WechatMsg, r *wechat.WechatReplyContent) interface{} {
reply := &wechat.MsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
}
switch r.ReplyType {
case wechat.ReplyTypeText:
return &wechat.TextMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
Content: r.Reply.Content,
}

case wechat.ReplyTypeImage:
return &wechat.ImageMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
Image: wechat.ImageContent{
MediaId: r.Reply.MediaId,
},
}
case wechat.ReplyTypeVoice:
return &wechat.VoiceMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
Voice: wechat.VoiceContent{
MediaId: r.Reply.MediaId,
},
}
case wechat.ReplyTypeVideo:
return &wechat.VideoMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
Video: wechat.VideoContent{
MediaId: r.Reply.MediaId,
Title: r.Reply.Title,
Description: r.Reply.Description,
},
}
case wechat.ReplyTypeMusic:
return &wechat.MusicMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
Music: wechat.MusicContent{
Title: r.Reply.Title,
Description: r.Reply.Description,
MusicUrl: r.Reply.MusicUrl,
HQMusicUrl: r.Reply.HQMusicUrl,
ThumbMediaId: r.Reply.ThumbMediaId,
},
}
case wechat.ReplyTypeNews:
return &wechat.NewsMsgReply{
ToUserName: msg.FromUserName,
FromUserName: msg.ToUserName,
CreateTime: time.Now().Unix(),
MsgType: r.ReplyType,
ArticleCount: len(r.Reply.Articles),
Articles: wechat.ArticleItem{
Item: r.Reply.Articles},
}

}
return reply
}

+ 13
- 10
routers/home.go View File

@@ -350,19 +350,22 @@ func ExploreDatasets(ctx *context.Context) {
var datasetsIds []int64
if ownerID > 0 {

datasetsIds = models.GetCollaboratorDatasetIdsByUserID(ownerID)
collaboratorDatasetsIds := models.GetCollaboratorDatasetIdsByUserID(ownerID)
teamDatasetsIds := models.GetTeamDatasetIdsByUserID(ownerID)
datasetsIds = append(collaboratorDatasetsIds, teamDatasetsIds...)
}

opts := &models.SearchDatasetOptions{
Keyword: keyword,
IncludePublic: true,
SearchOrderBy: orderBy,
Category: category,
Task: task,
License: license,
OwnerID: ownerID,
DatasetIDs: datasetsIds,
RecommendOnly: ctx.QueryBool("recommend"),
Keyword: keyword,
IncludePublic: true,
SearchOrderBy: orderBy,
Category: category,
Task: task,
License: license,
OwnerID: ownerID,
DatasetIDs: datasetsIds,
RecommendOnly: ctx.QueryBool("recommend"),
CloudBrainType: -1,
ListOptions: models.ListOptions{
Page: page,
PageSize: 30,


+ 150
- 36
routers/repo/cloudbrain.go View File

@@ -2,6 +2,7 @@ package repo

import (
"bufio"
"code.gitea.io/gitea/modules/grampus"
"encoding/json"
"errors"
"fmt"
@@ -186,7 +187,7 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error {
ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled

ctx.Data["cloudbraintype"] = models.TypeCloudBrainOne
ctx.Data["datasetType"] = models.TypeCloudBrainOne

ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode")

@@ -207,27 +208,14 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
image := strings.TrimSpace(form.Image)
uuid := form.Attachment
uuids := form.Attachment
jobType := form.JobType
gpuQueue := form.GpuType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
resourceSpecId := form.ResourceSpecId
branchName := form.BranchName
repo := ctx.Repo.Repository

tpl := tplCloudBrainNew
command := cloudbrain.Command
if jobType == string(models.JobTypeTrain) {
tpl = tplCloudBrainTrainJobNew
commandTrain, err := getTrainJobCommand(form)
if err != nil {
log.Error("getTrainJobCommand failed: %v", err)
ctx.RenderWithErr(err.Error(), tpl, &form)
return
}

command = commandTrain
}

tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
if err == nil {
@@ -273,6 +261,27 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
}
}

datasetInfos, datasetNames, err := models.GetDatasetInfo(uuids)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
}

command := cloudbrain.Command
if jobType == string(models.JobTypeTrain) {
tpl = tplCloudBrainTrainJobNew
commandTrain, err := getTrainJobCommand(form)
if err != nil {
log.Error("getTrainJobCommand failed: %v", err)
ctx.RenderWithErr(err.Error(), tpl, &form)
return
}

command = commandTrain
}

if branchName == "" {
branchName = cloudbrain.DefaultBranchName
}
@@ -285,11 +294,33 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {

commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params,
commitID, 0, 0, resourceSpecId)
req := cloudbrain.GenerateCloudBrainTaskReq{
Ctx: ctx,
DisplayJobName: displayJobName,
JobName: jobName,
Image: image,
Command: command,
Uuids: uuids,
DatasetNames: datasetNames,
DatasetInfos: datasetInfos,
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
JobType: jobType,
GpuQueue: gpuQueue,
Description: form.Description,
BranchName: branchName,
BootFile: form.BootFile,
Params: form.Params,
CommitID: commitID,
BenchmarkTypeID: 0,
BenchmarkChildTypeID: 0,
ResourceSpecId: resourceSpecId,
}

err = cloudbrain.GenerateTask(req)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tpl, &form)
@@ -574,12 +605,6 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
}

}
attachment, err := models.GetAttachmentByUUID(task.Uuid)
if err == nil {
ctx.Data["datasetname"] = attachment.Name
} else {
ctx.Data["datasetname"] = ""
}

ctx.Data["task"] = task
ctx.Data["jobName"] = task.JobName
@@ -1036,6 +1061,7 @@ func GetPublicImages(ctx *context.Context) {
IncludeOfficialOnly: ctx.QueryBool("recommend"),
SearchOrderBy: "type desc, num_stars desc,id desc",
Status: models.IMAGE_STATUS_SUCCESS,
CloudbrainType: ctx.QueryInt("cloudbrainType"),
}

getImages(ctx, &opts)
@@ -1471,7 +1497,34 @@ func SyncCloudbrainStatus() {
} else {
log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
}
} else if task.Type == models.TypeC2Net {
result, err := grampus.GetJob(task.JobID)
if err != nil {
log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
continue
}

if result != nil {
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
}
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
task.Duration = result.JobInfo.RunSec
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)

if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
task.EndTime = task.StartTime.Add(task.Duration)
}
task.CorrectCreateUnix()
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
continue
}
}
} else {
log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
}
@@ -1982,11 +2035,42 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo
//return
}

err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, childInfo.Attachment, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, cloudbrain.DefaultBranchName, "", "",
"", benchmarkTypeID, benchmarkChildTypeID, resourceSpecId)
uuid := childInfo.Attachment
datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form)
return
}

req := cloudbrain.GenerateCloudBrainTaskReq{
Ctx: ctx,
DisplayJobName: displayJobName,
JobName: jobName,
Image: image,
Command: command,
Uuids: uuid,
DatasetNames: datasetNames,
DatasetInfos: datasetInfos,
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
JobType: string(models.JobTypeBenchmark),
GpuQueue: gpuQueue,
Description: form.Description,
BranchName: cloudbrain.DefaultBranchName,
BootFile: "",
Params: "",
CommitID: "",
BenchmarkTypeID: benchmarkTypeID,
BenchmarkChildTypeID: benchmarkChildTypeID,
ResourceSpecId: resourceSpecId,
}

err = cloudbrain.GenerateTask(req)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
@@ -2080,11 +2164,41 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
}

err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params,
"", 0, benchmarkChildTypeID, resourceSpecId)
datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
}

req := cloudbrain.GenerateCloudBrainTaskReq{
Ctx: ctx,
DisplayJobName: displayJobName,
JobName: jobName,
Image: image,
Command: command,
Uuids: uuid,
DatasetNames: datasetNames,
DatasetInfos: datasetInfos,
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
JobType: jobType,
GpuQueue: gpuQueue,
Description: form.Description,
BranchName: branchName,
BootFile: form.BootFile,
Params: form.Params,
CommitID: "",
BenchmarkTypeID: 0,
BenchmarkChildTypeID: benchmarkChildTypeID,
ResourceSpecId: resourceSpecId,
}

err = cloudbrain.GenerateTask(req)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tpl, &form)


+ 78
- 0
routers/repo/dataset.go View File

@@ -410,6 +410,84 @@ func MyDatasets(ctx *context.Context) {
})
}

func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
page := ctx.QueryInt("page")
cloudbrainType := ctx.QueryInt("type")
keyword := strings.Trim(ctx.Query("q"), " ")
orderBy := models.SearchOrderByRecentUpdated
opts.Keyword = keyword
opts.SearchOrderBy = orderBy
opts.RecommendOnly = ctx.QueryBool("recommend")
opts.CloudBrainType = cloudbrainType
opts.ListOptions = models.ListOptions{
Page: page,
PageSize: setting.UI.DatasetPagingNum,
}
opts.NeedAttachment = true
opts.JustNeedZipFile = true
opts.User = ctx.User

datasets, count, err := models.SearchDataset(opts)

if err != nil {
ctx.ServerError("datasets", err)
return
}

data, err := json.Marshal(datasets)
if err != nil {
log.Error("json.Marshal failed:", err.Error())
ctx.JSON(200, map[string]string{
"result_code": "-1",
"error_msg": err.Error(),
"data": "",
})
return
}
ctx.JSON(200, map[string]string{
"result_code": "0",
"data": string(data),
"count": strconv.FormatInt(count, 10),
})

}

func CurrentRepoDatasetMultiple(ctx *context.Context) {

opts := &models.SearchDatasetOptions{
RepoID: ctx.Repo.Repository.ID,
}
datasetMultiple(ctx, opts)

}

func MyDatasetsMultiple(ctx *context.Context) {

opts := &models.SearchDatasetOptions{
UploadAttachmentByMe: true,
}
datasetMultiple(ctx, opts)

}

func PublicDatasetMultiple(ctx *context.Context) {

opts := &models.SearchDatasetOptions{
PublicOnly: true,
}
datasetMultiple(ctx, opts)

}

func MyFavoriteDatasetMultiple(ctx *context.Context) {

opts := &models.SearchDatasetOptions{
StarByMe: true,
DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID),
}
datasetMultiple(ctx, opts)
}

func PublicDataset(ctx *context.Context) {
page := ctx.QueryInt("page")
cloudbrainType := ctx.QueryInt("type")


+ 843
- 0
routers/repo/grampus.go View File

@@ -0,0 +1,843 @@
package repo

import (
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/grampus"
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
"encoding/json"
"errors"
"github.com/unknwon/com"
"io/ioutil"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)

const (
tplGrampusTrainJobShow base.TplName = "repo/grampus/trainjob/show"

//GPU
tplGrampusTrainJobGPUNew base.TplName = "repo/grampus/trainjob/gpu/new"

//NPU
tplGrampusTrainJobNPUNew base.TplName = "repo/grampus/trainjob/npu/new"
)

func GrampusTrainJobGPUNew(ctx *context.Context) {
ctx.Data["datasetType"] = models.TypeCloudBrainOne
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
if err != nil {
ctx.ServerError("get new train-job info failed", err)
return
}
ctx.HTML(http.StatusOK, tplGrampusTrainJobGPUNew)
}

func GrampusTrainJobNPUNew(ctx *context.Context) {
ctx.Data["datasetType"] = models.TypeCloudBrainTwo
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
if err != nil {
ctx.ServerError("get new train-job info failed", err)
return
}
ctx.HTML(200, tplGrampusTrainJobNPUNew)
}

func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error {
ctx.Data["PageIsCloudBrain"] = true

t := time.Now()
var displayJobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
ctx.Data["display_job_name"] = displayJobName

//get valid images
images, err := grampus.GetImages(processType)
if err != nil {
log.Error("GetImages failed:", err.Error())
} else {
ctx.Data["images"] = images.Infos
}

grampus.InitSpecialPool()

ctx.Data["GPUEnabled"] = true
ctx.Data["NPUEnabled"] = true
includeCenters := make(map[string]struct{})
excludeCenters := make(map[string]struct{})
if grampus.SpecialPools != nil {
for _, pool := range grampus.SpecialPools.Pools {
if pool.IsExclusive {
if !IsUserInOrgPool(ctx.User.ID, pool) {
ctx.Data[pool.Type+"Enabled"] = false
}
} else {
if strings.Contains(strings.ToLower(processType), strings.ToLower(pool.Type)) {
if IsUserInOrgPool(ctx.User.ID, pool) {
for _, center := range pool.Pool {
includeCenters[center.Queue] = struct{}{}
}
} else {
for _, center := range pool.Pool {
excludeCenters[center.Queue] = struct{}{}
}

}

}

}
}
}

//get valid resource specs
specs, err := grampus.GetResourceSpecs(processType)

grampusSpecs := getFilterSpecBySpecialPool(specs, includeCenters, excludeCenters)

if err != nil {
log.Error("GetResourceSpecs failed:", err.Error())
} else {
ctx.Data["flavor_infos"] = grampusSpecs
}

//get branches
branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil {
log.Error("GetBranches error:", err.Error())
} else {
ctx.Data["branches"] = branches
}

ctx.Data["branchName"] = ctx.Repo.BranchName

if processType == grampus.ProcessorTypeGPU {
ctx.Data["datasetType"] = models.TypeCloudBrainOne
} else if processType == grampus.ProcessorTypeNPU {
ctx.Data["datasetType"] = models.TypeCloudBrainTwo
}

return nil
}

func getFilterSpecBySpecialPool(specs *models.GetGrampusResourceSpecsResult, includeCenters map[string]struct{}, excludeCenters map[string]struct{}) []models.GrampusSpec {
if len(includeCenters) == 0 && len(excludeCenters) == 0 {
return specs.Infos
}
var grampusSpecs []models.GrampusSpec
for _, info := range specs.Infos {
if isInIncludeCenters(info, includeCenters) || (len(excludeCenters) != 0 && isNotAllInExcludeCenters(info, excludeCenters)) {
grampusSpecs = append(grampusSpecs, info)
}

}
return grampusSpecs
}

func isInIncludeCenters(grampusSpec models.GrampusSpec, centers map[string]struct{}) bool {
for _, center := range grampusSpec.Centers {
if _, ok := centers[center.ID]; ok {
return true
}
}
return false
}
func isNotAllInExcludeCenters(grampusSpec models.GrampusSpec, centers map[string]struct{}) bool {
for _, center := range grampusSpec.Centers {
if _, ok := centers[center.ID]; !ok {
return true
}
}
return false
}

func IsUserInOrgPool(userId int64, pool *models.SpecialPool) bool {
org, _ := models.GetOrgByName(pool.Org)
if org != nil {
isOrgMember, _ := models.IsOrganizationMember(org.ID, userId)
return isOrgMember
}
return false
}

func grampusParamCheckCreateTrainJob(form auth.CreateGrampusTrainJobForm) error {
if !strings.HasSuffix(strings.TrimSpace(form.BootFile), ".py") {
log.Error("the boot file(%s) must be a python file", form.BootFile)
return errors.New("启动文件必须是python文件")
}

if form.BranchName == "" {
log.Error("the branch must not be null!", form.BranchName)
return errors.New("代码分支不能为空!")
}

return nil
}

func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) {
displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
uuid := form.Attachment
description := form.Description
bootFile := strings.TrimSpace(form.BootFile)
params := form.Params
repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + cloudbrain.CodeMountPath + "/"
codeMinioPath := setting.CBCodePathPrefix + jobName + cloudbrain.CodeMountPath + "/"
dataMinioPath := setting.Attachment.Minio.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid
branchName := form.BranchName
flavorName := form.FlavorName
image := strings.TrimSpace(form.Image)

if !jobNamePattern.MatchString(displayJobName) {
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobGPUNew, &form)
return
}

errStr := checkSpecialPool(ctx, "GPU")
if errStr != "" {
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr(errStr, tplGrampusTrainJobGPUNew, &form)
return
}

//check count limit
count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.GPUResource)
if err != nil {
log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("system error", tplGrampusTrainJobGPUNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplGrampusTrainJobGPUNew, &form)
return
}
}

//check param
if err := grampusParamCheckCreateTrainJob(form); err != nil {
log.Error("paramCheckCreateTrainJob failed:(%v)", err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr(err.Error(), tplGrampusTrainJobGPUNew, &form)
return
}

//check whether the task name in the project is duplicated
tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeTrain), displayJobName)
if err == nil {
if len(tasks) != 0 {
log.Error("the job name did already exist", ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("the job name did already exist", tplGrampusTrainJobGPUNew, &form)
return
}
} else {
if !models.IsErrJobNotExist(err) {
log.Error("system error, %v", err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("system error", tplGrampusTrainJobGPUNew, &form)
return
}
}

//check dataset
attachment, err := models.GetAttachmentByUUID(uuid)
if err != nil {
log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobGPUNew, &form)
return
}

//prepare code and out path
_, err = ioutil.ReadDir(codeLocalPath)
if err == nil {
os.RemoveAll(codeLocalPath)
}

if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil {
log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form)
return
}

//todo: upload code (send to file_server todo this work?)
//upload code
if err := uploadCodeToMinio(codeLocalPath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form)
return
}

modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
if err := mkModelPath(modelPath); err != nil {
log.Error("Failed to mkModelPath: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form)
return
}

//init model readme
if err := uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/"); err != nil {
log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form)
return
}

//prepare command
command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name)
if err != nil {
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobGPUNew, &form)
return
}

commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

req := &grampus.GenerateTrainJobReq{
JobName: jobName,
DisplayJobName: displayJobName,
ComputeResource: models.GPUResource,
ProcessType: grampus.ProcessorTypeGPU,
Command: command,
ResourceSpecId: form.FlavorID,
ImageUrl: image,
Description: description,
BootFile: bootFile,
Uuid: uuid,
CommitID: commitID,
BranchName: branchName,
Params: form.Params,
FlavorName: flavorName,
EngineName: image,
DatasetName: attachment.Name,
IsLatestVersion: modelarts.IsLatestVersion,
VersionCount: modelarts.VersionCount,
WorkServerNumber: 1,
}

err = grampus.GenerateTrainJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error(), ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
ctx.RenderWithErr(err.Error(), tplGrampusTrainJobGPUNew, &form)
return
}
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func checkSpecialPool(ctx *context.Context, resourceType string) string {
grampus.InitSpecialPool()
if grampus.SpecialPools != nil {
for _, pool := range grampus.SpecialPools.Pools {

if pool.IsExclusive && pool.Type == resourceType {

org, _ := models.GetOrgByName(pool.Org)
if org != nil {
isOrgMember, _ := models.IsOrganizationMember(org.ID, ctx.User.ID)
if !isOrgMember {
return ctx.Tr("repo.grampus.no_operate_right")
}
}
}

}

}
return ""
}

func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) {
displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
uuid := form.Attachment
description := form.Description
bootFile := strings.TrimSpace(form.BootFile)
params := form.Params
repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
codeObsPath := grampus.JobPath + jobName + modelarts.CodePath
dataObsPath := setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
branchName := form.BranchName
isLatestVersion := modelarts.IsLatestVersion
flavorName := form.FlavorName
versionCount := modelarts.VersionCount
engineName := form.EngineName

if !jobNamePattern.MatchString(displayJobName) {
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobNPUNew, &form)
return
}

errStr := checkSpecialPool(ctx, "NPU")
if errStr != "" {
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr(errStr, tplGrampusTrainJobGPUNew, &form)
return
}

//check count limit
count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.NPUResource)
if err != nil {
log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplGrampusTrainJobNPUNew, &form)
return
}
}

//check param
if err := grampusParamCheckCreateTrainJob(form); err != nil {
log.Error("paramCheckCreateTrainJob failed:(%v)", err)
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form)
return
}

//check whether the task name in the project is duplicated
tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeTrain), displayJobName)
if err == nil {
if len(tasks) != 0 {
log.Error("the job name did already exist", ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("the job name did already exist", tplGrampusTrainJobNPUNew, &form)
return
}
} else {
if !models.IsErrJobNotExist(err) {
log.Error("system error, %v", err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form)
return
}
}

//check dataset
attachment, err := models.GetAttachmentByUUID(uuid)
if err != nil {
log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobNPUNew, &form)
return
}

//prepare code and out path
_, err = ioutil.ReadDir(codeLocalPath)
if err == nil {
os.RemoveAll(codeLocalPath)
}

if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil {
log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err)
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("Create task failed, server timed out", tplGrampusTrainJobNPUNew, &form)
return
}

//todo: upload code (send to file_server todo this work?)
if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil {
log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err)
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("Failed to obsMkdir_output", tplGrampusTrainJobNPUNew, &form)
return
}

if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("Failed to uploadCodeToObs", tplGrampusTrainJobNPUNew, &form)
return
}

//prepare command
command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name)
if err != nil {
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr("Create task failed, internal error", tplGrampusTrainJobNPUNew, &form)
return
}

commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

req := &grampus.GenerateTrainJobReq{
JobName: jobName,
DisplayJobName: displayJobName,
ComputeResource: models.NPUResource,
ProcessType: grampus.ProcessorTypeNPU,
Command: command,
ResourceSpecId: form.FlavorID,
ImageId: form.ImageID,
DataUrl: dataObsPath,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
BootFile: bootFile,
WorkServerNumber: form.WorkServerNumber,
Uuid: uuid,
CommitID: commitID,
IsLatestVersion: isLatestVersion,
BranchName: branchName,
Params: form.Params,
FlavorName: flavorName,
EngineName: engineName,
VersionCount: versionCount,
TotalVersionCount: modelarts.TotalVersionCount,
DatasetName: attachment.Name,
}

err = grampus.GenerateTrainJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form)
return
}
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func GrampusStopJob(ctx *context.Context) {
var ID = ctx.Params(":jobid")
var resultCode = "0"
var errorMsg = ""
var status = ""

task := ctx.Cloudbrain
for {
if task.Status == string(models.GrampusStatusStopped) || task.Status == string(models.GrampusStatusFailed) || task.Status == string(models.GrampusStatusSucceeded) {
log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
resultCode = "-1"
errorMsg = "system error"
break
}

res, err := grampus.StopJob(task.JobID)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
resultCode = strconv.Itoa(res.ErrorCode)
errorMsg = res.ErrorMsg
break
}

task.Status = string(models.GrampusStatusStopped)
if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow()
}
task.ComputeAndSetDuration()
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
resultCode = "-1"
errorMsg = "system error"
break
}

status = task.Status
break
}

ctx.JSON(200, map[string]interface{}{
"result_code": resultCode,
"error_msg": errorMsg,
"status": status,
"id": ID,
"StatusOK": 0,
})
}

func GrampusTrainJobDel(ctx *context.Context) {
var listType = ctx.Query("listType")
if err := deleteGrampusJob(ctx); err != nil {
log.Error("deleteGrampusJob failed: %v", err, ctx.Data["msgID"])
ctx.ServerError(err.Error(), err)
return
}

var isAdminPage = ctx.Query("isadminpage")
var isHomePage = ctx.Query("ishomepage")
if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
} else if isHomePage == "true" {
ctx.Redirect(setting.AppSubURL + "/cloudbrains")
} else {
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType)
}
}

func deleteGrampusJob(ctx *context.Context) error {
task := ctx.Cloudbrain

if task.Status != string(models.GrampusStatusStopped) && task.Status != string(models.GrampusStatusSucceeded) && task.Status != string(models.GrampusStatusFailed) {
log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
return errors.New("the job has not been stopped")
}

err := models.DeleteJob(task)
if err != nil {
log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"])
return err
}

storageType := models.TypeCloudBrainOne
if task.ComputeResource == models.NPUResource {
storageType = models.TypeCloudBrainTwo
}
deleteJobStorage(task.JobName, storageType)

return nil
}

func GrampusTrainJobShow(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

var task *models.Cloudbrain
task, err := models.GetCloudbrainByJobIDWithDeleted(ctx.Params(":jobid"))
if err != nil {
log.Error("GetCloudbrainByJobID failed:" + err.Error())
ctx.ServerError("system error", err)
return
}

if task.DeletedAt.IsZero() { //normal record
result, err := grampus.GetJob(task.JobID)
if err != nil {
log.Error("GetJob failed:" + err.Error())
//ctx.ServerError("GetJob failed", err)
//return
}

if result != nil {
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
}
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
if task.Status != result.JobInfo.Status || result.JobInfo.Status == models.GrampusStatusRunning {
task.Duration = result.JobInfo.RunSec
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)

if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
task.EndTime = task.StartTime.Add(task.Duration)
}
task.CorrectCreateUnix()
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob failed:" + err.Error())
}
}
}
}

if len(task.Parameters) > 0 {
var parameters models.Parameters
err := json.Unmarshal([]byte(task.Parameters), &parameters)
if err != nil {
log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
ctx.ServerError("system error", err)
return
}

if len(parameters.Parameter) > 0 {
paramTemp := ""
for _, Parameter := range parameters.Parameter {
param := Parameter.Label + " = " + Parameter.Value + "; "
paramTemp = paramTemp + param
}
task.Parameters = paramTemp[:len(paramTemp)-2]
} else {
task.Parameters = ""
}
}

taskList := make([]*models.Cloudbrain, 0)
taskList = append(taskList, task)
ctx.Data["version_list_task"] = taskList

ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
ctx.Data["displayJobName"] = task.DisplayJobName

aiCenterInfo := strings.Split(task.AiCenter, "+")
if len(aiCenterInfo) == 2 {
ctx.Data["ai_center"] = aiCenterInfo[1]
}

ctx.HTML(http.StatusOK, tplGrampusTrainJobShow)
}

func GrampusGetLog(ctx *context.Context) {
jobID := ctx.Params(":jobid")
job, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID failed: %v", err, ctx.Data["MsgID"])
ctx.ServerError(err.Error(), err)
return
}

content, err := grampus.GetTrainJobLog(job.JobID)
if err != nil {
log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"])
ctx.ServerError(err.Error(), err)
return
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobName": job.JobName,
"Content": content,
})

return
}

func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) {
var command string

workDir := grampus.NpuWorkDir
if processorType == grampus.ProcessorTypeGPU {
workDir = grampus.GpuWorkDir
}

command += "pwd;cd " + workDir + grampus.CommandPrepareScript
//download code & dataset
if processorType == grampus.ProcessorTypeNPU {
commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"
command += commandDownload
} else if processorType == grampus.ProcessorTypeGPU {
commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"
command += commandDownload
}

//check download result
commandCheckRes := "bash -c \"[[ $? -eq 0 ]] && exit 0 || exit -1;\";"
command += commandCheckRes

//unzip code & dataset
toolUnzip := "unzip -q '"
if strings.HasSuffix(datasetName, ".tar.gz") {
toolUnzip = "tar -zxvf '"
}
commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + toolUnzip + datasetName + "';"
command += commandUnzip

//check unzip result
commandCheckRes = "bash -c \"[[ $? -eq 0 ]] && exit 0 || exit -1;\";"
command += commandCheckRes

command += "echo \"unzip finished;start to exec code;\";"

//exec code
var parameters models.Parameters
var paramCode string
param := make([]models.Parameter, 0)
if len(paramSrc) != 0 {
err := json.Unmarshal([]byte(paramSrc), &parameters)
if err != nil {
log.Error("Failed to Unmarshal params: %s (%v)", paramSrc, err)
return command, err
}

for _, parameter := range parameters.Parameter {
param = append(param, models.Parameter{
Label: parameter.Label,
Value: parameter.Value,
})
paramCode += " --" + parameter.Label + "=" + parameter.Value
}
}

commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";"
command += commandCode

//get exec result
commandGetRes := "result=$?;"
command += commandGetRes

//upload models
if processorType == grampus.ProcessorTypeNPU {
commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;"
command += commandUpload
} else if processorType == grampus.ProcessorTypeGPU {
commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;"
command += commandUpload
}

//check exec result
commandCheckRes = "bash -c \"[[ $result -eq 0 ]] && exit 0 || exit -1\""
command += commandCheckRes

return command, nil
}

func downloadZipCode(ctx *context.Context, codePath, branchName string) error {
archiveType := git.ZIP
archivePath := codePath

if !com.IsDir(archivePath) {
if err := os.MkdirAll(archivePath, os.ModePerm); err != nil {
log.Error("MkdirAll failed:" + err.Error())
return err
}
}

// Get corresponding commit.
var (
commit *git.Commit
err error
)

gitRepo := ctx.Repo.GitRepo
if err != nil {
log.Error("OpenRepository failed:" + err.Error())
return err
}

if gitRepo.IsBranchExist(branchName) {
commit, err = gitRepo.GetBranchCommit(branchName)
if err != nil {
log.Error("GetBranchCommit failed:" + err.Error())
return err
}
}

archivePath = path.Join(archivePath, grampus.CodeArchiveName)
if !com.IsFile(archivePath) {
if err := commit.CreateArchive(archivePath, git.CreateArchiveOpts{
Format: archiveType,
Prefix: setting.Repository.PrefixArchiveFiles,
}); err != nil {
log.Error("CreateArchive failed:" + err.Error())
return err
}
}

return nil
}

+ 154
- 45
routers/repo/modelarts.go View File

@@ -146,7 +146,7 @@ func notebookNewDataPrepare(ctx *context.Context) error {
}
ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo

ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -285,14 +285,37 @@ func NotebookShow(ctx *context.Context) {
}
}

datasetDownloadLink := ""
datasetDownload := make([]models.DatasetDownload, 0)
if ctx.IsSigned {
if task.Uuid != "" && task.UserID == ctx.User.ID {
attachment, err := models.GetAttachmentByUUID(task.Uuid)
if err == nil {
task.DatasetName = attachment.Name
datasetDownloadLink = attachment.S3DownloadURL()
uuidList := strings.Split(task.Uuid, ";")
for _, uuidStr := range uuidList {
attachment, err := models.GetAttachmentByUUID(uuidStr)
if err != nil {
log.Error("GetAttachmentByUUID failed:%v", err.Error())
return
}
dataset, err := models.GetDatasetByID(attachment.DatasetID)
if err != nil {
log.Error("GetDatasetByID failed:%v", err.Error())
return
}
repo, err := models.GetRepositoryByID(dataset.RepoID)
if err != nil {
log.Error("GetRepositoryByID failed:%v", err.Error())
return
}
datasetDownload = append(datasetDownload, models.DatasetDownload{
DatasetName: attachment.Name,
DatasetDownloadLink: attachment.S3DownloadURL(),
RepositoryLink: repo.Link() + "/datasets",
})

}
// datasetName, err := GetDatasetNameByUUID(task.Uuid)
// if err == nil {
// task.DatasetName = datasetName
// }
}
}
user, err := models.GetUserByID(task.UserID)
@@ -324,7 +347,7 @@ func NotebookShow(ctx *context.Context) {
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
}
ctx.Data["duration"] = task.TrainJobDuration
ctx.Data["datasetDownloadLink"] = datasetDownloadLink
ctx.Data["datasetDownload"] = datasetDownload
ctx.Data["task"] = task
ctx.Data["ID"] = ID
ctx.Data["jobName"] = task.JobName
@@ -559,24 +582,11 @@ func TrainJobIndex(ctx *context.Context) {
}

listType := ctx.Query("listType")
if len(listType) == 0 {
listType = models.AllResource
}
ctx.Data["ListType"] = listType

typeCloudBrain := models.TypeCloudBrainAll
if listType == models.GPUResource {
typeCloudBrain = models.TypeCloudBrainOne
} else if listType == models.NPUResource {
typeCloudBrain = models.TypeCloudBrainTwo
} else if listType == models.AllResource {
typeCloudBrain = models.TypeCloudBrainAll
if listType == models.AllResource {
listType = ""
}
//else {
// log.Error("listType(%s) error", listType)
// ctx.ServerError("listType error", errors.New("listType error"))
// return
//}

var jobTypes []string
jobTypes = append(jobTypes, string(models.JobTypeTrain))
@@ -586,10 +596,11 @@ func TrainJobIndex(ctx *context.Context) {
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: typeCloudBrain,
JobTypeNot: false,
JobTypes: jobTypes,
IsLatestVersion: modelarts.IsLatestVersion,
ComputeResource: listType,
Type: models.TypeCloudBrainAll,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
@@ -599,11 +610,6 @@ func TrainJobIndex(ctx *context.Context) {
for i, task := range tasks {
tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain)
if task.Cloudbrain.Type == models.TypeCloudBrainOne {
tasks[i].ComputeResource = models.GPUResource
} else if task.Cloudbrain.Type == models.TypeCloudBrainTwo {
tasks[i].ComputeResource = models.NPUResource
}
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
@@ -690,7 +696,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error {
return err
}
ctx.Data["config_list"] = configList.ParaConfigs
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -764,7 +770,7 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts
ctx.Data["bootFile"] = form.BootFile
ctx.Data["uuid"] = form.Attachment
ctx.Data["branch_name"] = form.BranchName
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -858,7 +864,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
ctx.Data["uuid"] = task.Uuid
ctx.Data["flavor_code"] = task.FlavorCode
ctx.Data["engine_id"] = task.EngineID
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
if err != nil {
@@ -955,7 +961,7 @@ func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai
return err
}
ctx.Data["config_list"] = configList.ParaConfigs
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -979,7 +985,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/"
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
// dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
branch_name := form.BranchName
isLatestVersion := modelarts.IsLatestVersion
FlavorName := form.FlavorName
@@ -1095,6 +1101,27 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
Value: modelarts.Ascend,
})
}
datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid)
if err != nil {
log.Error("Failed to getDatasUrlListByUUIDS: %v", err)
trainJobErrorNewDataPrepare(ctx, form)
ctx.RenderWithErr("Failed to getDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobNew, &form)
return
}
dataPath := dataUrl
jsondatas, err := json.Marshal(datasUrlList)
if err != nil {
log.Error("Failed to Marshal: %v", err)
trainJobErrorNewDataPrepare(ctx, form)
ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobNew, &form)
return
}
if isMultiDataset {
param = append(param, models.Parameter{
Label: modelarts.MultiDataUrl,
Value: string(jsondatas),
})
}

//save param config
if isSaveParam == "on" {
@@ -1161,6 +1188,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
EngineName: EngineName,
VersionCount: VersionCount,
TotalVersionCount: modelarts.TotalVersionCount,
DatasetName: datasetNames,
}

//将params转换Parameters.Parameter,出错时返回给前端
@@ -1222,7 +1250,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/"
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/"
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
// dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
branch_name := form.BranchName
PreVersionName := form.VersionName
FlavorName := form.FlavorName
@@ -1314,6 +1342,28 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
})
}

datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid)
if err != nil {
log.Error("Failed to getDatasUrlListByUUIDS: %v", err)
versionErrorDataPrepare(ctx, form)
ctx.RenderWithErr("Failed to getDatasUrlListByUUIDS:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
return
}
dataPath := dataUrl
jsondatas, err := json.Marshal(datasUrlList)
if err != nil {
log.Error("Failed to Marshal: %v", err)
versionErrorDataPrepare(ctx, form)
ctx.RenderWithErr("json error:"+err.Error(), tplModelArtsTrainJobVersionNew, &form)
return
}
if isMultiDataset {
param = append(param, models.Parameter{
Label: modelarts.MultiDataUrl,
Value: string(jsondatas),
})
}

//save param config
if isSaveParam == "on" {
saveparams := append(param, models.Parameter{
@@ -1386,6 +1436,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
EngineName: EngineName,
PreVersionName: PreVersionName,
TotalVersionCount: latestTask.TotalVersionCount + 1,
DatasetName: datasetNames,
}

err = modelarts.GenerateTrainJobVersion(ctx, req, jobID)
@@ -2111,7 +2162,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error {
New: MODEL_LATEST,
})
ctx.Data["MODEL_COUNT"] = model_count
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -2177,7 +2228,7 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel
ctx.Data["model_version"] = form.ModelVersion
ctx.Data["ckpt_name"] = form.CkptName
ctx.Data["train_url"] = form.TrainUrl
ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo
ctx.Data["datasetType"] = models.TypeCloudBrainTwo

return nil
}
@@ -2247,24 +2298,35 @@ func ModelDownload(ctx *context.Context) {
err error
)

var jobID = ctx.Params(":jobid")
jobID := ctx.Params(":jobid")
versionName := ctx.Query("version_name")
parentDir := ctx.Query("parent_dir")
fileName := ctx.Query("file_name")
task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error())
return
}

path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir, fileName), "/")

url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
if err != nil {
log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("GetObsCreateSignedUrl", err)
return
var url string
if task.ComputeResource == models.NPUResource {
path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir, fileName), "/")
url, err = storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
if err != nil {
log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("GetObsCreateSignedUrl", err)
return
}
} else if task.ComputeResource == models.GPUResource {
filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir
url, err = storage.Attachments.PresignedGetURL(filePath, fileName)
if err != nil {
log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("PresignedGetURL", err)
return
}
}

ctx.Resp.Header().Set("Cache-Control", "max-age=0")
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}
@@ -2420,3 +2482,50 @@ func TrainJobDownloadLogFile(ctx *context.Context) {
ctx.Resp.Header().Set("Cache-Control", "max-age=0")
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}
func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) {
var isMultiDataset bool
var dataUrl string
var datasetNames string
var datasUrlList []models.Datasurl
uuids := strings.Split(uuidStr, ";")
if len(uuids) > setting.MaxDatasetNum {
log.Error("the dataset count(%d) exceed the limit", len(uuids))
return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("the dataset count exceed the limit")
}

datasetInfos := make(map[string]models.DatasetInfo)
attachs, err := models.GetAttachmentsByUUIDs(uuids)
if err != nil {
log.Error("GetAttachmentsByUUIDs failed: %v", err)
return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("GetAttachmentsByUUIDs failed")
}
for i, attach := range attachs {
fileName := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(attach.Name, ".zip"), ".tar.gz"), ".tgz")
for _, datasetInfo := range datasetInfos {
if fileName == datasetInfo.Name {
log.Error("the dataset name is same: %v", attach.Name)
return datasUrlList, dataUrl, datasetNames, isMultiDataset, errors.New("the dataset name is same")
}
}
if len(attachs) <= 1 {
dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + attach.UUID + "/"
isMultiDataset = false
} else {
dataUrl = "/" + setting.Bucket + "/" + setting.BasePath + path.Join(attachs[0].UUID[0:1], attachs[0].UUID[1:2]) + "/" + attachs[0].UUID + attachs[0].UUID + "/"
datasetUrl := "s3://" + setting.Bucket + "/" + setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + attach.UUID + "/"
datasUrlList = append(datasUrlList, models.Datasurl{
DatasetUrl: datasetUrl,
DatasetName: fileName,
})
isMultiDataset = true
}

if i == 0 {
datasetNames = attach.Name
} else {
datasetNames += ";" + attach.Name
}
}

return datasUrlList, dataUrl, datasetNames, isMultiDataset, nil
}

+ 23
- 0
routers/routes/routes.go View File

@@ -1041,6 +1041,11 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/public_datasets", repo.PublicDataset)
m.Get("/my_favorite", repo.MyFavoriteDataset)

m.Get("/current_repo_m", repo.CurrentRepoDatasetMultiple)
m.Get("/my_datasets_m", repo.MyDatasetsMultiple)
m.Get("/public_datasets_m", repo.PublicDatasetMultiple)
m.Get("/my_favorite_m", repo.MyFavoriteDatasetMultiple)

m.Group("/status", func() {
m.Get("/:uuid", repo.GetDatasetStatus)
})
@@ -1101,6 +1106,24 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
})
}, context.RepoRef())
m.Group("/grampus", func() {
m.Group("/train-job", func() {
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.GrampusTrainJobShow)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.GrampusStopJob)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload)
})
m.Group("/gpu", func() {
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.GrampusTrainJobGPUNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobGpuCreate)
})
m.Group("/npu", func() {
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.GrampusTrainJobNPUNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobNpuCreate)
})
})
}, context.RepoRef())
m.Group("/modelmanage", func() {
m.Post("/create_model", reqRepoModelManageWriter, repo.SaveModel)
m.Post("/create_new_model", repo.SaveNewNameModel)


+ 2
- 7
routers/user/home.go View File

@@ -769,12 +769,6 @@ func Cloudbrains(ctx *context.Context) {
if page <= 0 {
page = 1
}
debugType := models.TypeCloudBrainAll
if listType == models.GPUResource {
debugType = models.TypeCloudBrainOne
} else if listType == models.NPUResource {
debugType = models.TypeCloudBrainTwo
}

var jobTypes []string
jobTypeNot := false
@@ -821,7 +815,6 @@ func Cloudbrains(ctx *context.Context) {
},
Keyword: keyword,
UserID: ctxUser.ID,
Type: debugType,
JobTypeNot: jobTypeNot,
JobStatusNot: jobStatusNot,
JobStatus: jobStatuses,
@@ -829,6 +822,8 @@ func Cloudbrains(ctx *context.Context) {
NeedRepoInfo: true,
IsLatestVersion: modelarts.IsLatestVersion,
RepoIDList: repoIDList,
ComputeResource: listType,
Type: models.TypeCloudBrainAll,
})
if err != nil {
ctx.ServerError("Get job failed:", err)


+ 1
- 0
routers/user/profile.go View File

@@ -263,6 +263,7 @@ func Profile(ctx *context.Context) {
Page: page,
PageSize: setting.UI.ExplorePagingNum,
},
CloudBrainType: -1,
}

if len(datasetSearchOptions.SearchOrderBy) == 0 {


+ 1
- 1
services/socketwrap/clientManager.go View File

@@ -10,7 +10,7 @@ import (
"github.com/elliotchance/orderedmap"
)

var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}

type ClientsManager struct {
Clients *orderedmap.OrderedMap


+ 3
- 3
templates/admin/cloudbrain/list.tmpl View File

@@ -102,7 +102,7 @@
</a>
{{else if eq .JobType "TRAIN"}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts{{end}}/train-job/{{$JobID}}"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 0}}/cloudbrain{{else if eq .Cloudbrain.Type 1}}/modelarts{{else if eq .Cloudbrain.Type 2}}/grampus{{end}}/train-job/{{$JobID}}"
title="{{.DisplayJobName}}" style="font-size: 14px;">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -204,7 +204,7 @@
{{else}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
class="ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED"}}disabled {{else}} blue {{end}}button"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain/train-job{{else}}/modelarts/{{if eq .JobType "INFERENCE"}}inference-job{{else}}train-job{{end}}{{end}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{else}}/modelarts/{{if eq .JobType "INFERENCE"}}inference-job{{else}}train-job{{end}}{{end}}"
data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
{{$.i18n.Tr "repo.stop"}}
</a>
@@ -212,7 +212,7 @@
</div>
<!-- 删除任务 -->
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}/train-job{{end}}/{{$JobID}}/del?isadminpage=true'
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{end}}/{{$JobID}}/del?isadminpage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"


+ 4
- 4
templates/custom/select_dataset.tmpl View File

@@ -1,9 +1,9 @@
<div class="dataset-repolink" id="dataset-repolink-init" style="display: none;" data-repolink="{{.RepoLink}}"
data-cloudranin-type="{{.cloudbraintype}}"></div>
<div class="inline {{if eq .cloudbraintype 0}} required {{end}} field" id="dataset-base">
data-dataset-type="{{.datasetType}}"></div>
<div class="inline {{if eq .datasetType 0}} required {{end}} field" id="dataset-base">
<label>{{.i18n.Tr "dataset.dataset"}}</label>
<input type="hidden" name="attachment" :value="dataset_uuid">
{{if eq .cloudbraintype 0}}
{{if eq .datasetType 0}}
<input class="disabled" type="text" :value="dataset_name" placeholder="{{.i18n.Tr "cloudbrain.select_dataset"}}"
required onfocus="this.blur();">
{{else}}
@@ -18,7 +18,7 @@
<input type="text" placeholder="{{.i18n.Tr "dataset.search_dataset"}}" v-model="searchDataItem">
</div>

<el-tabs v-model="activeName" @tab-click="handleClick('{{.RepoLink}}',activeName,{{.cloudbraintype}})">
<el-tabs v-model="activeName" @tab-click="handleClick('{{.RepoLink}}',activeName,{{.datasetType}})">
<el-tab-pane label="{{.i18n.Tr "dataset.current_project"}}" name="first">
<div style="display: flex;align-items: center;justify-content: space-between;padding: 1rem 0;border-bottom:1px solid #F5F5F5"
v-for="(dataset,index) in currentRepoDataset" :key="index">


+ 4
- 4
templates/custom/select_dataset_train.tmpl View File

@@ -1,11 +1,11 @@
<div class="dataset-repolink" id="dataset-repolink-init" style="display: none;" data-repolink="{{.RepoLink}}"
data-cloudranin-type="{{.cloudbraintype}}"></div>
<div class="inline required unite min_title field" id="dataset-base" style="margin-bottom: 0 !important;">
data-dataset-type="{{.datasetType}}"></div>
<div class="inline required min_title field" id="dataset-base" style="margin-bottom: 0 !important;">
{{if or (.benchmarkMode) (.newInference)}}
<label
style="font-weight: normal;">{{if .benchmarkMode}}{{.i18n.Tr "repo.model_manager"}}</label><span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span>{{else}}{{.i18n.Tr "dataset.dataset"}}</label>&nbsp;&nbsp;&nbsp;&nbsp;{{end}}
{{else}}
<label style="font-weight: normal;">{{.i18n.Tr "dataset.dataset"}}</label>&nbsp;&nbsp;&nbsp;
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "dataset.dataset"}}</label>
{{end}}
<input type="hidden" name="attachment" :value="dataset_uuid">
<input class="disabled" type="text" :value="dataset_name" required onfocus="this.blur();" style="width: 48.5%;">
@@ -22,7 +22,7 @@
<input type="text" placeholder="{{.i18n.Tr "dataset.search_dataset"}}" v-model="searchDataItem">
</div>

<el-tabs v-model="activeName" @tab-click="handleClick('{{.RepoLink}}',activeName,{{.cloudbraintype}})">
<el-tabs v-model="activeName" @tab-click="handleClick('{{.RepoLink}}',activeName,{{.datasetType}})">
<el-tab-pane label="{{.i18n.Tr "dataset.current_project"}}" name="first" v-loading="loadingDataIndex">
<div style="display: flex;align-items: center;justify-content: space-between;padding: 1rem 0;border-bottom:1px solid #F5F5F5"
v-for="(dataset,index) in currentRepoDataset" :key="index">


+ 1
- 1
templates/explore/datasets.tmpl View File

@@ -154,7 +154,7 @@
<a class="{{if eq .SortType "downloadtimes"}}active{{end}} item"
href="{{$.Link}}?sort=downloadtimes&q={{$.Keyword}}&tab={{$.TabName}}&category={{$.Category}}&task={{$.Task}}&license={{$.License}}&recommend={{$.Recommend}}">{{.i18n.Tr "repo.issues.filter_sort.downloadtimes"}}</a>
<a class="{{if eq .SortType "moststars"}}active{{end}} item"
href="{{$.Link}}?sort=moststars&q={{$.Keyword}}&tab={{$.TabName}}&category={{$.Category}}&task={{$.Task}}&license={{$.License}}&recommend={{$.Recommend}}">{{.i18n.Tr "repo.issues.filter_sort.moststars"}}</a>
href="{{$.Link}}?sort=moststars&q={{$.Keyword}}&tab={{$.TabName}}&category={{$.Category}}&task={{$.Task}}&license={{$.License}}&recommend={{$.Recommend}}">{{.i18n.Tr "dataset.collection_num"}}</a>
<a class="{{if eq .SortType "mostusecount"}}active{{end}} item"
href="{{$.Link}}?sort=mostusecount&q={{$.Keyword}}&tab={{$.TabName}}&category={{$.Category}}&task={{$.Task}}&license={{$.License}}&recommend={{$.Recommend}}">{{.i18n.Tr "repo.issues.filter_sort.mostusecount"}}</a>
</div>


+ 1
- 1
templates/repo/cloudbrain/benchmark/show.tmpl View File

@@ -474,7 +474,7 @@

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{$.datasetname}}
{{.DatasetName}}
</div>
</td>
</tr>


+ 5
- 5
templates/repo/cloudbrain/new.tmpl View File

@@ -104,6 +104,7 @@
top: 14px;
z-index: 2; */
}

</style>

<div id="mask">
@@ -119,8 +120,8 @@
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">

<div class="column">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
{{template "base/alert" .}}
<div class="ui negative message" id="messageInfo">
<p></p>
@@ -218,8 +219,10 @@
<div id="images-new-cb">

</div>
<div id="select-multi-dataset">

{{template "custom/select_dataset" .}}
</div>
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown"
@@ -278,8 +281,6 @@
</div>
</div>
</form>


</div>
</div>
</div>
@@ -373,5 +374,4 @@
$('#store_category').attr("value", selected_value)
})


</script>

+ 1
- 1
templates/repo/cloudbrain/show.tmpl View File

@@ -409,7 +409,7 @@
<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-BenchmarkTypeName">
{{$.datasetname}}
{{.DatasetName}}
</div>
</td>
</tr>


+ 59
- 48
templates/repo/cloudbrain/trainjob/new.tmpl View File

@@ -1,20 +1,14 @@
{{template "base/head" .}}
<style>
.unite {
.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title {
font-size: 16px !important;
padding-left: 3rem !important;
}

.min_title {
.min_title{
font-size: 14px !important;
padding-left: 6rem !important;
margin-bottom: 2rem !important;

}

.width {
@@ -33,9 +27,10 @@

.width85 {
width: 85% !important;
margin-left: 4.5rem !important;
margin-left: 10.5rem !important;
align-items: center;
}
.width81 {
margin-left: 1.5rem !important;
width: 81% !important;
@@ -60,6 +55,14 @@
text-align: center;
color: #C2C7CC;
}
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 14px !important;
}

</style>
<!-- <div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
@@ -76,6 +79,7 @@
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
@@ -87,9 +91,22 @@
<input type="hidden" name="action" value="update">
<input type="hidden" id="ai_engine_name" name="engine_names" value="">
<input type="hidden" id="ai_flaver_name" name="flaver_names" value="">
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
@@ -110,18 +127,18 @@
Ascend NPU</a>
</div>
</div>
<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name"
placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}"
tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required
maxlength="64">
<span class="tooltips" style="display: block;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
<span class="tooltips" style="display: block;margin-left: 11.5rem;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
</div>

<div class="unite min_title inline field">
<label style="font-weight: normal;"
for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;"
for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255"
placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}
onchange="this.value=this.value.substring(0, 255)"
@@ -130,11 +147,11 @@
</div>
<div class="ui divider"></div>

<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>


<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version" name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
@@ -162,8 +179,8 @@
</select>
</div>

<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search width806 dropdown" placeholder="选择GPU类型"
style='width:385px' name="gpu_type">
{{range .train_gpu_types}}
@@ -171,27 +188,12 @@
{{end}}
</select>
</div>

<!-- <div class="required unite min_title inline field" style="position: relative;">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}&nbsp;</label>
<input class="width81" type="text" list="cloudbrain_image" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" name="image" required autofocus maxlength="255">
<i class="times circle outline icon icons" style="visibility: hidden;" onclick="clearValue()"></i>
<datalist class="ui search" id="cloudbrain_image" name="image">
{{range .images}}
<option name="image" value="{{.Place}}">{{.PlaceView}}</option>
{{end}}
{{range .public_images}}
<option name="image" value="{{.Place}}">{{.PlaceView}}</option>
{{end}}
</datalist>
</div> -->

<div id="images-new-cb">

</div>

<div class="inline unite min_title field required">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<div class="inline field min_title required">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
{{if .bootFile}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}"
tabindex="3" autofocus required maxlength="255">
@@ -206,13 +208,17 @@
</span>
<a href="https://git.openi.org.cn/OpenIOSSG/MNIST_PytorchExample_GPU" target="_blank">查看样例</a>
</div>
<div id="select-multi-dataset">

</div>


{{template "custom/select_dataset_train" .}}
<!-- {{template "custom/select_dataset_train" .}} -->
<span class="tooltips"
style="margin-left: 11.5rem;margin-bottom: 2rem;">训练脚本存储在/code中,数据集存储在/dataset中,训练输出请存储在/model中以供后续下载。</span>
<div class="inline unite min_title field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
style="margin-left: 11.5rem;margin-bottom: 1rem;">训练脚本存储在/code中,数据集存储在/dataset中,训练输出请存储在/model中以供后续下载。</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para"
style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i
class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
@@ -239,8 +245,8 @@
</div>
</div>

<div class="required inline unite min_title field">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown" placeholder="选择资源规格"
style='width:385px' name="resource_spec_id">
{{range .train_resource_specs}}
@@ -250,7 +256,8 @@
</select>
</div>

<div class="inline unite min_title field">
<div class="inline field" style="padding: 1rem 0;">
<label class="label-fix-width"></label>
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
@@ -274,7 +281,11 @@

$('.menu .item')
.tab();

$(document).keydown(function(event){
switch(event.keyCode){
case 13:return false;
}
});
let sever_num = $('#trainjob_work_server_num')
$('.add').click(function () {
sever_num.val(parseInt(sever_num.val()) + 1)


+ 4
- 0
templates/repo/cloudbrain/trainjob/show.tmpl View File

@@ -474,6 +474,10 @@
<div id="dir_list{{.VersionName}}">

</div>
<div style="display:flex;align-items: center;justify-content: end;color: #f2711c;">
<i class="ri-error-warning-line" style="margin-right:0.5rem;"></i>
<span>{{$.i18n.Tr "repo.file_limit_100"}}</span>
</div>
</div>

</div>


+ 440
- 0
templates/repo/grampus/trainjob/gpu/new.tmpl View File

@@ -0,0 +1,440 @@
{{template "base/head" .}}
<style>

.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{
font-size: 14px !important;
margin-bottom: 2rem !important;
}
.width{
width:100% !important;
}
.width80{
width: 80.7% !important;
margin-left: 10px;
}
.width806{
width: 80.6% !important;
margin-left: -2px;
}
.width85{
width: 85% !important;
margin-left: 10.5rem !important;
align-items: center;
}
.width81{
width: 81% !important;
}

.add{font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 0px 5px 5px 0px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.min{
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 5px 0px 0px 5px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 14px !important;
}
</style>
<!-- <div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div> -->
<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<input type="hidden" id="ai_engine_name" name="engine_names" value="">
<input type="hidden" id="ai_flavor_name" name="flavor_name" value="">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="active item" href="{{.RepoLink}}/grampus/train-job/{{if.NPUEnabled}}npu{{else}}gpu{{end}}/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU
</a>
<a {{if.NPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
Ascend NPU</a>
</div>
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}" tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required maxlength="64">
<span class="tooltips" style="margin-left: 11.5rem;display: block;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
</div>
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
</div>
<div class="ui divider"></div>

<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>

<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version" name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>

<div id="images-new-grampus">
</div>

<div class="inline min_title field required">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
{{if .bootFile}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}" tabindex="3" autofocus required maxlength="255" >
{{else}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" >
{{end}}
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
<a href="https://git.openi.org.cn/OpenIOSSG/MNIST_PytorchExample_GPU/src/branch/master/train_for_c2net.py" target="_blank">查看样例</a>
</div>

{{template "custom/select_dataset_train" .}}
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.grampus.gpu_dataset_path_rule"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field" style="margin-top: 1rem;">
{{if .params}}
{{if ne 0 (len .params)}}
{{range $k ,$v := .params}}
<div class="two fields width85" id="para{{$k}}">
<div class="field">
<input type="text" name="shipping_first-name" value={{$v.Label}} required>
</div>
<div class="field">
<input type="text" name="shipping_last-name" value={{$v.Value}} required>
</div>
<span>
<i class="trash icon"></i>
</span>

</div>
{{end}}
{{end}}
{{end}}
</div>
</div>

<div class="required min_title inline field" id="flavor_name">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.ID}}">{{.Name}}</option>
{{end}}
</select>
</div>
<div class="inline min_title field">
<label class="label-fix-width"></label>
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="{{.RepoLink}}/modelarts/train-job">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
<!-- 模态框 -->
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
//let url_href = window.location.pathname.split('create')[0]
//$(".ui.button").attr('href',url_href)
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

let sever_num = $('#trainjob_work_server_num')
$('.add').click(function(){
sever_num.val(parseInt(sever_num.val())+1)
if(sever_num.val()>=26){
sever_num.val(parseInt(sever_num.val())-1)
}
})
$('.min').click(function(){
sever_num.val(parseInt(sever_num.val())-1)
if(sever_num.val()<=0){
sever_num.val(parseInt(sever_num.val())+1)
}
})
// 参数增加、删除、修改、保存
function Add_parameter(i){
value = '<div class="two fields width85" id= "para'+ i +'">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" required placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" required placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
var len = $(".dynamic.field .two.fields").length
$(".dynamic.field .two.fields").each(function(){
var cur_index = $(this).index()
$(this).attr('id', 'para' + cur_index)
})
});

$('.ui.parameter.green.button').click(function(){
var parameters = [];
$('table tr').each(function() {
$(this).find('td:eq(1)').each(function(){
parameters.push($(this).text());
})
$(this).find('input').each(function(){
parameters.push($(this).text())
})
});
$('.ui.parameter.modal')
.modal('hide');
for(var i = 2; i < parameters.length; i++){
switch(i) {
// 数据集uuid待完成
// case (2):
// console.log(1)
// break;
// $("#trainjob_datasets").val(parameters[i]);
// console.log($("#trainjob_datasets").val())
case (3):
$("input[name='boot_file']").val(parameters[i]);
break;
case (4):
var para = parameters[i].split(" ")
for(var j = 0; j < para.length; j++){
var para_name = para[j].split('=')[0]
var para_value = para[j].split('=')[1]
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
var pid = 'para' + len
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_first-name]").val(para_name)
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_last-name]").val(para_value)
}
break;
// 数据集pool_id待完成
// case (5):
// $("select[name='pool_id']").val(parameters[i]);
// break;
case (6):
$("input[name='work_server_number']").val(parameters[i]);
break;
}
}
})

$('.ui.save.checkbox').click(function(){
$(this).checkbox({
onChange: function(){
if ($('.ui.save.checkbox').checkbox('is checked')){
$('#save_para').removeClass("disabled")
}else{
$('#save_para').addClass("disabled")
}
}
});
})

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$(".item.active.parameter_config").click(function(){
$('.ui.parameter.modal')
.modal('setting', 'closable', false)
.modal('show');
})

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})
$('select.dropdown')
.dropdown();

function validate(){
$('.ui.form')
.form({
on: 'blur',
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
}
]
},
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[a-zA-Z0-9_]$/]',
}
]
},
attachment:{
identifier : 'attachment',
rules: [
{
type: 'empty',
}
]

},
flavor:{
identifier : 'flavor',
rules: [
{
type: 'empty',
}
]

},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
}
]
}
},
onSuccess: function(){
// $('.ui.page.dimmer').dimmer('show')
document.getElementById("mask").style.display = "block"
},
onFailure: function(e){
return false;
}
})
}
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}
function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}
function get_name(){
let name1=$("#engine_name .text").text()
let name2=$("#flavor_name .text").text()
$("input#ai_engine_name").val(name1)
$("input#ai_flavor_name").val(name2)

}
$('.ui.create_train_job.green.button').click(function(e) {
get_name()
send_run_para()
validate()
})
</script>

+ 431
- 0
templates/repo/grampus/trainjob/npu/new.tmpl View File

@@ -0,0 +1,431 @@
{{template "base/head" .}}
<style>
.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{
font-size: 14px !important;
margin-bottom: 2rem !important;
}
.width{
width:100% !important;
}
.width80{
width: 80.7% !important;
margin-left: 10px;
}
.width85{
width: 85% !important;
margin-left: 10.5rem !important;
align-items: center;
}
.width81{
width: 81% !important;
}

.add{font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 0px 5px 5px 0px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.min{
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 5px 0px 0px 5px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 14px !important;
}
</style>
<!-- <div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div> -->
<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<input type="hidden" id="ai_engine_name" name="engine_name" value="">
<input type="hidden" id="ai_flavor_name" name="flavor_name" value="">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="active item" href="{{.RepoLink}}/grampus/train-job/{{if.NPUEnabled}}npu{{else}}gpu{{end}}/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU
</a>
<a {{if.NPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
Ascend NPU</a>
</div>
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}" tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required maxlength="64">
<span class="tooltips" style="display: block;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
</div>

<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
</div>
<div class="ui divider"></div>

<h4 class="train-job-title title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>


<div class="required unite min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version" name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>

<div class="required min_title inline field" id="engine_name">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select class="ui dropdown width81" id="trainjob_images" name="image_id">
{{range .images}}
<option name="image_id" value="{{.ID}}">{{.Name}}</option>
{{end}}
</select>
</div>

<div class="inline min_title field required">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
{{if .bootFile}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}" tabindex="3" autofocus required maxlength="255" >
{{else}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" >
{{end}}
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
<a href="https://git.openi.org.cn/OpenIOSSG/MNIST_Example/src/branch/master/train_for_c2net.py" target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div>
{{template "custom/select_dataset_train" .}}
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.grampus.dataset_path_rule"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field" style="margin-top: 1rem;">
</div>
</div>

<div class="required min_title inline field" id="flavor_name">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.ID}}">{{.Name}}</option>
{{end}}
</select>
</div>
<div class="inline required min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>

<div class="ui labeled input" style="width: 5%;">

<input style="border-radius: 0;text-align: center;"type="hidden" name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255" value="1" readonly>
<div class="field" id="trainjob_work_server_num_select" name="work_server_number_select">
<select class="ui dropdown width" style='width: 100%;' name="work_server_id">
<option name="server_id" value="1">1</option>
</select>
</div>

</div>
</div>

<div class="inline min_title field">
<label class="label-fix-width"></label>
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="{{.RepoLink}}/modelarts/train-job">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>

<!-- 模态框 -->

</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

// let sever_num = $("#trainjob_work_server_num_select .text").text() //$('#trainjob_work_server_num')
// console.log("sever_num:",sever_num)
// $('.add').click(function(){
// sever_num.val(parseInt(sever_num.val())+1)
// if(sever_num.val()>=26){
// sever_num.val(parseInt(sever_num.val())-1)
// }
// })
// $('.min').click(function(){
// sever_num.val(parseInt(sever_num.val())-1)
// if(sever_num.val()<=0){
// sever_num.val(parseInt(sever_num.val())+1)
// }
// })
// 参数增加、删除、修改、保存
function Add_parameter(i){
value = '<div class="two fields width85" id= "para'+ i +'">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" required placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" required placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
var len = $(".dynamic.field .two.fields").length
$(".dynamic.field .two.fields").each(function(){
var cur_index = $(this).index()
$(this).attr('id', 'para' + cur_index)
})
});

$('.ui.parameter.green.button').click(function(){
var parameters = [];
$('table tr').each(function() {
$(this).find('td:eq(1)').each(function(){
parameters.push($(this).text());
})
$(this).find('input').each(function(){
parameters.push($(this).text())
})

});
$('.ui.parameter.modal')
.modal('hide');
for(var i = 2; i < parameters.length; i++){
switch(i) {
// 数据集uuid待完成
// case (2):
// console.log(1)
// break;
// $("#trainjob_datasets").val(parameters[i]);
// console.log($("#trainjob_datasets").val())
case (3):
$("input[name='boot_file']").val(parameters[i]);
break;
case (4):
var para = parameters[i].split(" ")
for(var j = 0; j < para.length; j++){
var para_name = para[j].split('=')[0]
var para_value = para[j].split('=')[1]
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
var pid = 'para' + len
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_first-name]").val(para_name)
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_last-name]").val(para_value)
}
break;
// 数据集pool_id待完成
// case (5):
// $("select[name='pool_id']").val(parameters[i]);
// break;
case (6):
// $("input[name='work_server_number']").val(parameters[i]);
break;
}
}
})

$('.ui.save.checkbox').click(function(){
$(this).checkbox({
onChange: function(){
if ($('.ui.save.checkbox').checkbox('is checked')){
$('#save_para').removeClass("disabled")

}else{
$('#save_para').addClass("disabled")
}
}
});
})

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$(".item.active.parameter_config").click(function(){
$('.ui.parameter.modal')
.modal('setting', 'closable', false)
.modal('show');
})

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})
$('select.dropdown')
.dropdown();

function validate(){
$('.ui.form')
.form({
on: 'blur',
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
}
]
},
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-zA-Z0-9-_]{1,64}[a-zA-Z0-9_]$/]',
}
]
},
attachment:{
identifier : 'attachment',
rules: [
{
type: 'empty',
}
]

},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
}
]
}
},
onSuccess: function(){
// $('.ui.page.dimmer').dimmer('show')
document.getElementById("mask").style.display = "block"
},
onFailure: function(e){
return false;
}
})
}
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}
function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}
function get_name(){
let name1=$("#engine_name .text").text()
let name2=$("#flavor_name .text").text()
$("input#ai_engine_name").val(name1)
$("input#ai_flavor_name").val(name2)

let val_server_num_select = $("#trainjob_work_server_num_select .text").text()
// console.log("val_server_num_select:",val_server_num_select)
$("input#trainjob_work_server_num").val(val_server_num_select)

}
$('.ui.create_train_job.green.button').click(function(e) {
get_name()
send_run_para()
validate()
})
</script>

+ 982
- 0
templates/repo/grampus/trainjob/show.tmpl View File

@@ -0,0 +1,982 @@
{{template "base/head" .}}
<style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 360px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread {
margin-bottom: -0.5rem !important;
padding-left: 1rem;
padding-top: 0.5rem;
}
</style>
<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<h4 class="ui header" id="vertical-segment">
<div class="ui breadcrumb">
<a class="section" href="{{.RepoLink}}/debugjob?debugListType=all">
{{.i18n.Tr "repo.cloudbrain"}}
</a>
<div class="divider"> / </div>
<a class="section" href="{{$.RepoLink}}/modelarts/train-job">
{{$.i18n.Tr "repo.modelarts.train_job"}}
</a>
<div class="divider"> / </div>
<div class="active section">{{.displayJobName}}</div>
</div>
</h4>
{{range $k ,$v := .version_list_task}}
<div class="ui accordion border-according" id="accordion{{.VersionName}}"
data-repopath="{{$.RepoRelPath}}/modelarts/train-job" data-jobid="{{.JobID}}"
data-version="{{.VersionName}}">
<div class="{{if eq $k 0}}active{{end}} title padding0">
<div class="according-panel-heading">
<div class="accordion-panel-title">
<i class="dropdown icon"></i>
<span class="accordion-panel-title-content">
<span>
<div style="float: right;">
{{$.CsrfTokenHtml}}
</div>
<div class="ac-display-inblock title_text acc-margin-bottom">

<span class="cti-mgRight-sm">
{{if not (eq .StartTime 0)}}
{{TimeSinceUnix1 .StartTime}}
{{else}}
{{TimeSinceUnix1 .CreatedUnix}}
{{end}}</span>
<span class="cti-mgRight-sm">
{{$.i18n.Tr "repo.modelarts.current_version"}}:{{.VersionName}}</span>
<span class="cti-mgRight-sm">
{{$.i18n.Tr "repo.modelarts.parent_version"}}:{{.PreVersionName}}</span>
<span class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.status"}}:
<span id="{{.VersionName}}-status-span"><i id="icon"
style="vertical-align: middle;" class="{{.Status}}"></i><span id="text"
style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
</span>
<span
class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}:</span>
<span class="cti-mgRight-sm uc-accordionTitle-black"
id="{{.VersionName}}-duration-span">{{.TrainJobDuration}}</span>
<span data-tooltip="刷新" style="cursor: pointer;" data-inverted=""
onclick="refreshStatus({{.VersionName}})"><i
class="redo icon redo-color"></i></span>

</div>
</span>
</span>
</div>
</div>
</div>
<div class="{{if eq $k 0}}active{{end}} content">
<div class="content-pad">
<div class="ui pointing secondary menu" style="border-bottom: 1px solid rgba(34,36,38,.15);">

<a class="active item"
data-tab="first{{$k}}">{{$.i18n.Tr "repo.modelarts.train_job.config"}}</a>
<a class="item" data-tab="second{{$k}}"
onclick="loadLog({{.VersionName}})">{{$.i18n.Tr "repo.modelarts.log"}}</a>
<a class="item" data-tab="third{{$k}}"
onclick="loadModelFile({{.VersionName}},'','','init')">{{$.i18n.Tr "repo.model_download"}}</a>
</div>
<div class="ui tab active" data-tab="first{{$k}}">
<div style="padding-top: 10px;">
<div class="tab_2_content">
<div class="ac-grid ac-grid-col2">
<div class="ac-grid-col">
<table class="ti-form">
<tbody class="ti-text-form">
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_task"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.DisplayJobName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.status"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-status">
{{.Status}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.run_version"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.VersionName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.start_time"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
<span style="font-size: 12px;" class="">
{{if not (eq .StartTime 0)}}
{{TimeSinceUnix1 .StartTime}}
{{else}}
{{TimeSinceUnix1 .CreatedUnix}}
{{end}}</span>
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-duration">
{{.TrainJobDuration}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.standard"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.FlavorName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.compute_node"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.WorkServerNumber}}
</div>
</td>
</tr>
</tbody>
</table>
</div>
<div class="ac-grid-col">
<table class="ti-form">
<tbody class="ti-text-form">
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.EngineName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.code_version"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.BranchName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.start_file"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.BootFile}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.train_dataset"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.DatasetName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" title="{{.Parameters}}">
{{.Parameters}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.grampus.train_job.ai_center"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-ai_center">
{{$.ai_center}}
</div>
</td>
</tr>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.description"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
title="{{.Description}}">
{{.Description}}
</div>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>

</div>
</div>
<div class="ui tab" data-tab="second{{$k}}">
<div style="position: relative;">
<div class="ui message message{{.VersionName}}" style="display: none;">
<div id="header"></div>
</div>
<div class="ui attached log" id="log{{.VersionName}}"
style="height: 300px !important; overflow: auto;">
<input type="hidden" name="end_line" value>
<input type="hidden" name="start_line" value>
<pre id="log_file{{.VersionName}}"></pre>
</div>

</div>

</div>
<div class="ui tab" data-tab="third{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

</div>
<div id="dir_list{{.VersionName}}">

</div>
</div>

</div>
</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>
<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> {{.i18n.Tr "cloudbrain.delete_task"}}
</div>

<div class="content">
<p>{{.i18n.Tr "cloudbrain.task_delete_confirm"}}</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> {{.i18n.Tr "cloudbrain.operate_cancel"}}
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> {{.i18n.Tr "cloudbrain.operate_confirm"}}
</div>
</div>
</div>
</div>
<!-- 创建模型 -->
<div id="newmodel">
<div class="ui modal second">
<div class="header" style="padding: 1rem;background-color: rgba(240, 240, 240, 100);">
<h4 id="model_header">导入新模型</h4>
</div>
<div class="content content-padding">
<form id="formId" method="POST" class="ui form">
<div class="ui error message">
</div>
{{$.CsrfTokenHtml}}
<input type="hidden" name="trainTaskCreate" value="true">

<div class="two inline fields ">
<div class="required ten wide field">
<label style="margin-left: -23px;">选择训练任务</label>
<input type="hidden" class="width83" id="JobId" name="JobId" readonly required>
<input class="width83" id="JobName" readonly required>

</div>
<div class="required six widde field">
<label>版本</label>
<input class="width70" id="VersionName" name="VersionName" readonly required>
</div>
</div>

<div class="required inline field" id="modelname">
<label>模型名称</label>
<input style="width: 45%;" id="name" name="Name" required maxlength="25"
onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<div class="required inline field" id="verionname">
<label>模型版本</label>
<input style="width: 45%;" id="version" name="Version" value="" readonly required
maxlength="255">
</div>
<div class="inline field">
<label>模型标签</label>
<input style="width: 83%;margin-left: 7px;" id="label" name="Label" maxlength="255"
placeholder='{{.i18n.Tr "repo.modelarts.train_job.label_place"}}'>
</div>
<div class="inline field">
<label for="description">模型描述</label>
<textarea style="width: 83%;margin-left: 7px;" id="Description" name="Description" rows="3"
maxlength="255" placeholder='{{.i18n.Tr "repo.modelarts.train_job.new_place"}}'
onchange="this.value=this.value.substring(0, 255)"
onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 256)"></textarea>
</div>

<div class="inline field" style="margin-left: 75px;">
<button onclick="createModel()" type="button" class="ui create_train_job green button"
style="position: absolute;">
{{.i18n.Tr "repo.model.manage.sava_model"}}
</button>
</div>
</form>
<div class="actions" style="display: inline-block;margin-left: 180px;">
<button class="ui button cancel">{{.i18n.Tr "repo.cloudbrain.cancel"}}</button>
</div>
</div>


</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('.menu .item').tab()
$(document).ready(function () {
$('.ui.accordion').accordion({ selector: { trigger: '.icon' } });
});
$(document).ready(function () {
$('.secondary.menu .item').tab();
});

let userName
let repoPath
let jobID
let downlaodFlag = {{ $.canDownload }}
$(document).ready(function () {
let url = window.location.href;
let urlArr = url.split('/')
userName = urlArr.slice(-5)[0]
repoPath = urlArr.slice(-4)[0]
jobID = urlArr.slice(-1)[0]
})
function stopBubbling(e) {
e = window.event || e;
if (e.stopPropagation) {
e.stopPropagation(); //阻止事件 冒泡传播
} else {
e.cancelBubble = true; //ie兼容
}
}

function showcreate(obj) {
$('.ui.modal.second')
.modal({
centered: false,
onShow: function () {
$('input[name="Version"]').addClass('model_disabled')
// $('input[name="JobId"]').text(obj.JobName)
$('#JobName').val(obj.DisplayJobName).addClass('model_disabled')
$('input[name="JobId"]').val(obj.JobID)
$('input[name="VersionName"]').val(obj.VersionName).addClass('model_disabled')
$('.ui.dimmer').css({ "background-color": "rgb(136, 136, 136,0.7)" })
createModelName()
},
onHide: function () {
document.getElementById("formId").reset();
$('.ui.dimmer').css({ "background-color": "" })
$('.ui.error.message').text()
$('.ui.error.message').css('display', 'none')
}
})
.modal('show')
}
function createModel() {
let url_href = `/${userName}/${repoPath}/modelmanage/create_new_model`
let data = $("#formId").serialize()
$("#mask").css({ "display": "block", "z-index": "9999" })
$.ajax({
url: url_href,
type: 'POST',
data: data,
success: function (res) {
location.href = `/${userName}/${repoPath}/modelmanage/show_model`
$('.ui.modal.second').modal('hide')
},
error: function (xhr) {
// 隐藏 loading
// 只有请求不正常(状态码不为200)才会执行
$('.ui.error.message').text(xhr.responseText)
$('.ui.error.message').css('display', 'block')
},
complete: function (xhr) {
$("#mask").css({ "display": "none", "z-index": "1" })
}
})

}
function createModelName() {
let repoName = location.pathname.split('/')[2]
let modelName = repoName + '_model_' + Math.random().toString(36).substr(2, 4)
$('#name').val(modelName)
$('#version').val("0.0.1")
}
function renderSize(value) {
if (null == value || value == '') {
return "0 Bytes";
}
var unitArr = new Array("Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB");
var index = 0;
var srcsize = parseFloat(value);
index = Math.floor(Math.log(srcsize) / Math.log(1024));
var size = srcsize / Math.pow(1024, index);
size = size.toFixed(0);//保留的小数位数
return size + unitArr[index];
}
function refreshStatus(version_name) {
$.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}?version_name=${version_name}`, (data) => {
// header status and duration
$(`#${version_name}-duration-span`).text(data.JobDuration)
$(`#${version_name}-status-span span`).text(data.JobStatus)
$(`#${version_name}-status-span i`).attr("class", data.JobStatus)
// detail status and duration
$('#' + version_name + '-duration').text(data.JobDuration)
$('#' + version_name + '-status').text(data.JobStatus)
$('#' + version_name + '-ai_center').text(data.AiCenter)
loadLog(version_name)


}).fail(function (err) {
console.log(err);
});
stopBubbling(arguments.callee.caller.arguments[0])
}
function deleteVersion(version_name) {
stopBubbling(arguments.callee.caller.arguments[0])
let flag = 1;
$('.ui.basic.modal').modal({
onDeny: function () {
flag = false
},
onApprove: function () {
$.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/del_version`, { version_name: version_name }, (data) => {
if (data.VersionListCount === 0) {
location.href = `/${userName}/${repoPath}/modelarts/train-job`
} else {
$('#accordion' + version_name).remove()
}

}).fail(function (err) {
console.log(err);
});
flag = true
},
onHidden: function () {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')

}
function stopVersion(version_name) {
stopBubbling(arguments.callee.caller.arguments[0])
$.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/stop_version`, { version_name: version_name }, (data) => {
if (data.StatusOK === 0) {
$('#' + version_name + '-stop').addClass('disabled')
refreshStatus(version_name)
}
}).fail(function (err) {
console.log(err);
});
}
function loadLog(version_name) {
document.getElementById("mask").style.display = "block"
$.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}/log?version_name=${version_name}&lines=50&order=asc`, (data) => {
$('input[name=end_line]').val(data.EndLine)
$('input[name=start_line]').val(data.StartLine)
$(`#log_file${version_name}`).text(data.Content)
document.getElementById("mask").style.display = "none"
}).fail(function (err) {
document.getElementById("mask").style.display = "none"
console.log(err);
});
}
function loadModelFile(version_name, parents, filename, init) {
parents = parents || ''
filename = filename || ''
init = init || ''
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/model_list?version_name=${version_name}&parentDir=${parents}`, (data) => {
$(`#dir_list${version_name}`).empty()
renderDir(data, version_name)
if (init === "init") {
$(`input[name=model${version_name}]`).val("")
$(`input[name=modelback${version_name}]`).val(version_name)
$(`#file_breadcrumb${version_name}`).empty()
let htmlBread = ""
htmlBread += `<div class='active section'>${version_name}</div>`
htmlBread += "<div class='divider'> / </div>"
$(`#file_breadcrumb${version_name}`).append(htmlBread)
} else {
renderBrend(version_name, parents, filename, init)
}
}).fail(function (err) {
console.log(err, version_name);
});

}
function renderBrend(version_name, parents, filename, init) {
if (init == "folder") {
let htmlBrend = ""
let sectionName = $(`#file_breadcrumb${version_name} .active.section`).text()
let parents1 = $(`input[name=model${version_name}]`).val()
let filename1 = $(`input[name=modelback${version_name}]`).val()
if (parents1 === "") {
$(`#file_breadcrumb${version_name} .active.section`).replaceWith(`<a class='section' onclick="loadModelFile('${version_name}','${parents1}','','init')">${sectionName}</a>`)
} else {
$(`#file_breadcrumb${version_name} .active.section`).replaceWith(`<a class='section' onclick="loadModelFile('${version_name}','${parents1}','${filename1}')">${sectionName}</a>`)
}

htmlBrend += `<div class='active section'>${filename}</div>`
htmlBrend += "<div class='divider'> / </div>"
$(`#file_breadcrumb${version_name}`).append(htmlBrend)
$(`input[name=model${version_name}]`).val(parents)
$(`input[name=modelback${version_name}]`).val(filename)
} else {
$(`input[name=model${version_name}]`).val(parents)
$(`input[name=modelback${version_name}]`).val(filename)
$(`#file_breadcrumb${version_name} a.section:contains(${filename})`).nextAll().remove()
$(`#file_breadcrumb${version_name} a.section:contains(${filename})`).replaceWith(`<div class='active section'>${filename}</div>`)
$(`#file_breadcrumb${version_name} div.section:contains(${filename})`).append("<div class='divider'> / </div>")
}

}
function renderDir(data, version_name) {
let html = ""
html += "<div class='ui grid' style='margin:0;'>"
html += "<div class='row' style='padding: 0;'>"
html += "<div class='ui sixteen wide column' style='padding:1rem;'>"
html += "<div class='dir list'>"
html += "<table id='repo-files-table' class='ui single line table pad20'>"
html += '<tbody>'
// html += "</tbody>"
for (let i = 0; i < data.Dirs.length; i++) {
let dirs_size = renderSize(data.Dirs[i].Size)
html += "<tr>"
html += "<td class='name six wid'>"
html += "<span class='truncate'>"
html += "<span class='octicon octicon-file-directory'>"
html += "</span>"
if (data.Dirs[i].IsDir) {
html += `<a onclick="loadModelFile('${version_name}','${data.Dirs[i].ParenDir}','${data.Dirs[i].FileName}','folder')">`
html += "<span class='fitted'><i class='folder icon' width='16' height='16' aria-hidden='true'></i>" + data.Dirs[i].FileName + "</span>"
} else {
if (downlaodFlag) {
html += `<a href="${location.href}/model_download?version_name=${version_name}&file_name=${data.Dirs[i].FileName}&parent_dir=${data.Dirs[i].ParenDir}">`
}
else {
html += `<a class="disabled">`
}
html += "<span class='fitted'><i class='file icon' width='16' height='16' aria-hidden='true'></i>" + data.Dirs[i].FileName + "</span>"
}
html += '</a>'
html += "</span>"
html += "</td>"
html += "<td class='message seven wide'>"
if (data.Dirs[i].IsDir) {
html += "<span class='truncate has-emoji'></span>"
} else {
html += "<span class='truncate has-emoji'>" + `${dirs_size}` + "</span>"
}

html += "</td>"

html += "<td class='text right age three wide'>"
html += "<span class='truncate has-emoji'>" + data.Dirs[i].ModTime + "</span>"
html += "</td>"
html += "</tr>"

}
html += "</tbody>"
html += "</table>"
html += "</div>"
html += "</div>"
html += "</div>"
html += "</div>"
$(`#dir_list${version_name}`).append(html)
}
function debounce(fn, delay) {
let timer;
return (...args) => {
// 判断定时器是否存在,清除定时器
if (timer) {
clearTimeout(timer);
}

// 重新调用setTimeout
timer = setTimeout(() => {
fn.apply(this, args);
}, delay);
};
}
const fn = debounce(logScroll, 500)
function logScroll(version_name) {
let container = document.querySelector(`#log${version_name}`)
let scrollTop = container.scrollTop
let scrollHeight = container.scrollHeight
let clientHeight = container.clientHeight
let scrollLeft = container.scrollLeft
if (((parseInt(scrollTop) + clientHeight == scrollHeight || parseInt(scrollTop) + clientHeight + 1 == scrollHeight || parseInt(scrollTop) + clientHeight - 1 == scrollHeight)) && parseInt(scrollTop) !== 0 && scrollLeft == 0) {
let end_line = $(`#log${version_name} input[name=end_line]`).val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?version_name=${version_name}&base_line=${end_line}&lines=50&order=desc`, (data) => {
if (data.Lines == 0) {
$(`.message${version_name} #header`).text('您已翻阅至日志底部')
$(`.message${version_name}`).css('display', 'block')
setTimeout(function () {
$(`.message${version_name}`).css('display', 'none')
}, 1000)
} else {
if (end_line === data.EndLine) {
return
}
else {
$(`#log${version_name} input[name=end_line]`).val(data.EndLine)
$(`#log${version_name}`).append('<pre>' + data.Content)
}

}
}).fail(function (err) {
console.log(err);
});
}
if ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10].includes(scrollTop) && scrollLeft == 0) {
let start_line = $(`#log${version_name} input[name=start_line]`).val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?version_name=${version_name}&base_line=${start_line}&lines=50&order=asc`, (data) => {
if (data.Lines == 0) {
$(`.message${version_name} #header`).text('您已翻阅至日志顶部')
$(`.message${version_name}`).css('display', 'block')
setTimeout(function () {
$(`.message${version_name}`).css('display', 'none')
}, 1000)
} else {
$(`#log${version_name} input[name=start_line]`).val(data.StartLine) //如果变动就改变所对应的值
$(`#log${version_name}`).prepend('<pre>' + data.Content)
}
}).fail(function (err) {
console.log(err);
});
}
}
function scrollAnimation(dom, currentY, targetY, currentX) {
let needScrollTop = targetY - currentY;
let _currentY = currentY;
setTimeout(() => {
// 一次调用滑动帧数,每次调用会不一样
//取总距离的十分之一
const dist = Math.ceil(needScrollTop / 10);
_currentY += dist;
//移动一个十分之一
dom.scrollTo(currentX || 0, _currentY, 'smooth');
// 如果移动幅度小于十个像素,直接移动,否则递归调用,实现动画效果
if (needScrollTop > 10 || needScrollTop < -10) {
scrollAnimation(dom, _currentY, targetY)
} else {
dom.scrollTo(0, targetY, 'smooth')
}
}, 1)
}

$('.log_top').click(function () {
// let logContentDom = document.querySelector('.log')
// if(!logContentDom)
// return
// let version_name = $('.log_top').data('version')
let version_name = $(this).data('version')
let logContentDom = document.querySelector(`#log${version_name}`)

$(`#log_file${version_name}`).siblings('pre').remove()
$.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}/log?version_name=${version_name}&base_line=&lines=50&order=asc`, (data) => {

$(`#log${version_name} input[name=end_line]`).val(data.EndLine) //如果变动就改变所对应的值
$(`#log${version_name} input[name=start_line]`).val(data.StartLine)
$(`#log${version_name}`).prepend('<pre>' + data.Content)
$(`.message${version_name} #header`).text('您已翻阅至日志顶部')
$(`.message${version_name}`).css('display', 'block')
setTimeout(function () {
$(`.message${version_name}`).css('display', 'none')
}, 1000)
scrollAnimation(logContentDom, logContentDom.scrollTop, 0);
})

})
$('.log_bottom').click(function (e) {
let version_name = $(this).data('version')
let logContentDom = document.querySelector(`#log${version_name}`)
$(`#log_file${version_name}`).siblings('pre').remove()
$.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}/log?version_name=${version_name}&base_line=&lines=50&order=desc`, (data) => {

$(`#log${version_name} input[name=end_line]`).val(data.EndLine) //如果变动就改变所对应的值
$(`#log${version_name} input[name=start_line]`).val(data.StartLine)
$(`#log${version_name}`).append('<pre>' + data.Content)
$.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}/log?version_name=${version_name}&base_line=${data.EndLine}&lines=50&order=desc`, (data) => {
if (data.Lines == 0) {
$(`.message${version_name} #header`).text('您已翻阅至日志底部')
$(`.message${version_name}`).css('display', 'block')
setTimeout(function () {
$(`.message${version_name}`).css('display', 'none')
}, 1000)
} else {
if (end_line === data.EndLine) {
return
}
else {
$(`#log${version_name} input[name=end_line]`).val(data.EndLine)
$(`#log${version_name}`).append('<pre>' + data.Content)
}

}
}).fail(function (err) {
console.log(err);
});
scrollAnimation(logContentDom, logContentDom.scrollTop + 1, logContentDom.scrollHeight - logContentDom.clientHeight);
})
})
</script>

+ 4
- 1
templates/repo/modelarts/notebook/new.tmpl View File

@@ -18,6 +18,7 @@
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
{{template "base/alert" .}}
<div class="ui negative message" id="messageInfo">
<p></p>
@@ -59,7 +60,9 @@
{{end}}
</select>
</div>
{{template "custom/select_dataset" .}}
<div id="select-multi-dataset">

</div>

<!--<div class="inline required field">
<label>工作环境</label>


+ 32
- 44
templates/repo/modelarts/notebook/show.tmpl View File

@@ -69,7 +69,6 @@
}

.tab_2_content {
min-height: 460px;
margin-left: 10px;
}

@@ -338,30 +337,6 @@
</div>
</td>
</tr>

<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-duration">
{{$.duration}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.datasetdownload"}}
</td>

<td class="ti-text-form-content">
<div class="text-span-new" id="model_description">
{{$.datasetDownloadLink}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.description"}}
@@ -392,24 +367,6 @@
</div>
</td>
</tr>



<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dataset"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-BenchmarkTypeName">
{{.DatasetName}}
</div>
</td>
</tr>



<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.standard"}}
@@ -456,6 +413,18 @@
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-duration">
{{$.duration}}
</div>
</td>
</tr>


</tbody>
@@ -463,8 +432,26 @@
</div>
</div>

</div>

<div style="clear:both">
<table style="border:none" class="ui fixed small stackable table">
<thead>
<tr><th style="color: #8a8e99;font-size:12px" class="three wide center aligned">数据集文件</th>
<th style="color: #8a8e99;font-size:12px"class="eleven wide">数据集下载地址</th>
<th style="color: #8a8e99;font-size:12px" class="two wide center aligned">操作</th>
</tr></thead>
<tbody>
{{range $.datasetDownload}}
<tr>
<td style="word-wrap: break-word;word-break: break-all;"><a href="{{.RepositoryLink}}">{{.DatasetName}}</a></td>
<td style="word-wrap: break-word;word-break: break-all;">{{.DatasetDownloadLink}}</td>
<td class="center aligned"><a class="ui poping up clipboard" id="clipboard-btn" data-original="{{$.i18n.Tr "repo.copy_link"}}" data-success="{{$.i18n.Tr "repo.copy_link_success"}}" data-error="{{$.i18n.Tr "repo.copy_link_error"}}" data-content="{{$.i18n.Tr "repo.copy_link"}}" data-variation="inverted tiny" data-clipboard-text="{{.DatasetDownloadLink}}">复制链接</a></td>
</tr>
{{end}}
</tbody>
</table>
</div>
</div>
</div>

@@ -509,4 +496,5 @@
$(document).ready(function () {
$('.secondary.menu .item').tab();
});
console.log({{$.datasetDownload}})
</script>

+ 4
- 4
templates/repo/modelarts/trainjob/index.tmpl View File

@@ -112,7 +112,7 @@

<!-- 任务名 -->
<div class="three wide column padding0">
<a class="title" href='{{if eq .ComputeResource "NPU" }}{{$.Link}}/{{.JobID}}{{else}}{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}{{end}}' title="{{.DisplayJobName}}" style="font-size: 14px;">
<a class="title" href='{{if eq .Cloudbrain.Type 1 }}{{$.Link}}/{{.JobID}}{{else if eq .Cloudbrain.Type 0}}{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}{{else if eq .Cloudbrain.Type 2}}{{$.RepoLink}}/grampus/train-job/{{.JobID}}{{end}}' title="{{.DisplayJobName}}" style="font-size: 14px;">

<span class="fitted" style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
</a>
@@ -153,18 +153,18 @@
<div class="ui compact buttons">
{{$.CsrfTokenHtml}}
{{if .CanDel}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{.JobID}}" class="ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED"}}disabled {{else}} blue {{end}}button" data-repopath='{{$.RepoRelPath}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}' data-jobid="{{.JobID}}" data-version="{{.VersionName}}">
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{.JobID}}" class="ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED"}}disabled {{else}} blue {{end}}button" data-repopath='{{$.RepoRelPath}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}' data-jobid="{{.JobID}}" data-version="{{.VersionName}}">
{{$.i18n.Tr "repo.stop"}}
</a>
{{else}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{.JobID}}" class="ui basic disabled button">
<a class="ui basic disabled button">
{{$.i18n.Tr "repo.stop"}}
</a>
{{end}}

</div>
<!-- 删除任务 -->
<form class="ui compact buttons" id="delForm-{{.JobID}}" action='{{if eq .ComputeResource "NPU" }}{{$.Link}}/{{.JobID}}{{else}}{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}{{end}}/del' method="post">
<form class="ui compact buttons" id="delForm-{{.JobID}}" action='{{if eq .Cloudbrain.Type 1}}{{$.Link}}/{{.JobID}}{{else if eq .Cloudbrain.Type 0}}{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}{{else if eq .Cloudbrain.Type 2}}{{$.RepoLink}}/grampus/train-job/{{.JobID}}{{end}}/del' method="post">
<input type="hidden" name="listType" value="{{$.ListType}}">
{{$.CsrfTokenHtml}}
{{if .CanDel}}


+ 134
- 133
templates/repo/modelarts/trainjob/new.tmpl View File

@@ -1,59 +1,58 @@
{{template "base/head" .}}
<style>
.unite {
.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{
font-size: 14px !important;
margin-bottom: 2rem !important;
}
.width{
width:100% !important;
}
.width48{
width: 48.5% !important;
}
.width80{
width: 80.7% !important;
margin-left: 10px;
}
.width85{
width: 85% !important;
margin-left: 10.5rem !important;
align-items: center;
}
.width81{
margin-left: 1.5rem;
width: 81% !important;
}

.add{font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 0px 5px 5px 0px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.min{
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 5px 0px 0px 5px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title {
font-size: 16px !important;
padding-left: 3rem !important;
}

.min_title {
font-size: 14px !important;
padding-left: 6rem !important;
margin-bottom: 2rem !important;

}

.width {
width: 100% !important;
}

.width80 {
width: 80.7% !important;
margin-left: 10px;
}

.width85 {
width: 85% !important;
margin-left: 4.5rem !important;
}

.width81 {
margin-left: 1.5rem;
width: 81% !important;
}

.add {
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 0px 5px 5px 0px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}

.min {
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 5px 0px 0px 5px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
</style>
<!-- <div class="ui page dimmer">
@@ -71,6 +70,7 @@
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
@@ -82,9 +82,22 @@
<input type="hidden" name="action" value="update">
<input type="hidden" id="ai_engine_name" name="engine_names" value="">
<input type="hidden" id="ai_flaver_name" name="flaver_names" value="">
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="required inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/train-job/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
@@ -105,55 +118,46 @@
Ascend NPU</a>
</div>
</div>
<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name"
placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}"
tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required
maxlength="64">
<span class="tooltips" style="display: block;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
<div class="required inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}" tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required maxlength="64">
<span class="tooltips" style="margin-left:11.5rem;display: block;">{{.i18n.Tr "cloudbrain.job_name_rule"}}</span>
</div>

<div class="unite min_title inline field">
<label style="font-weight: normal;"
for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255"
placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}
onchange="this.value=this.value.substring(0, 255)"
onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)"></textarea>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
</div>
<div class="ui divider"></div>

<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required unite min_title inline field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version" name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>

<div class="required inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version" name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>



<div class="required unite min_title inline fields" style="width: 90%;">
<label
style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</label>
<div class="required inline min_title fields" style="width: 95%;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="field" style="flex: 1.5;">
<select class="ui dropdown width" id="trainjob_engines">
{{range .engines}}
@@ -163,44 +167,37 @@
</div>

<div class="field" style="flex: 2;" id="engine_name">
<select class="ui dropdown width" id="trainjob_engine_versions" style='width: 100%;'
name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
<select class="ui dropdown width" id="trainjob_engine_versions" name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>

</div>

</div>

<div class="inline unite min_title field required">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
{{if .bootFile}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}"
tabindex="3" autofocus required maxlength="255">
{{else}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3"
autofocus required maxlength="255">
{{end}}
<span>
<i class="question circle icon link"
data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}}
data-position="right center" data-variation="mini"></i>
</span>
<a href="https://git.openi.org.cn/OpenIOSSG/MINIST_Example"
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
<div class="inline field min_title required">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
{{if .bootFile}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="{{.bootFile}}" tabindex="3" autofocus required maxlength="255" >
{{else}}
<input style="width: 48.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" >
{{end}}
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
<a href="https://git.openi.org.cn/OpenIOSSG/MINIST_Example" target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div>
<div id="select-multi-dataset">

{{template "custom/select_dataset_train" .}}
<span class="tooltips"
style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "cloudbrain.dataset_path_rule"}}</span>
<div class="inline unite min_title field">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para"
style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i
class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;">{{.i18n.Tr "cloudbrain.dataset_path_rule"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field" style="margin-top: 1rem;">
{{if ne 0 (len .params)}}
{{range $k ,$v := .params}}
@@ -222,7 +219,7 @@
</div>


<div class="required field " style="display: none;">
<div class="required min_title field " style="display: none;">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
{{range .resource_pools}}
@@ -248,17 +245,16 @@
</div>
</div>

<div class="required unite min_title inline field" id="flaver_name">
<label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
<div class="required inline min_title field" id="flaver_name">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui dropdown width48" id="trainjob-flavor" name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.Code}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="inline required unite min_title field">
<label
style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
<div class="inline required min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>

<div class="ui labeled input" style="width: 5%;">

@@ -274,7 +270,8 @@
</div>
</div>

<div class="inline unite min_title field">
<div class="inline field" style="padding: 1rem 0;">
<label class="label-fix-width"></label>
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
@@ -297,7 +294,11 @@

$('.menu .item')
.tab();

$(document).keydown(function(event){
switch(event.keyCode){
case 13:return false;
}
});
// let sever_num = $("#trainjob_work_server_num_select .text").text() //$('#trainjob_work_server_num')
// console.log("sever_num:",sever_num)
// $('.add').click(function(){


+ 5
- 0
templates/repo/modelarts/trainjob/show.tmpl View File

@@ -525,6 +525,7 @@
data-version="{{.VersionName}}"></i>
<div id="metric-{{.VersionName}}" style="height: 260px;width: 870px;">
</div>
</div>
<div class="ui tab" data-tab="third{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1">
@@ -537,6 +538,10 @@
<div id="dir_list{{.VersionName}}">

</div>
<div style="display:flex;align-items: center;justify-content: end;color: #f2711c;">
<i class="ri-error-warning-line" style="margin-right:0.5rem;"></i>
<span>{{$.i18n.Tr "repo.file_limit_100"}}</span>
</div>
</div>

</div>


+ 3
- 3
templates/user/dashboard/cloudbrains.tmpl View File

@@ -94,7 +94,7 @@
</a>
{{else if eq .JobType "TRAIN"}}
<a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .ComputeResource "NPU"}}modelarts{{else}}cloudbrain{{end}}/train-job/{{$JobID}}"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .Cloudbrain.Type 1}}modelarts{{else if eq .Cloudbrain.Type 0}}cloudbrain{{else if eq .Cloudbrain.Type 2}}grampus{{end}}/train-job/{{$JobID}}"
title="{{.DisplayJobName}}" style="font-size: 14px;">
<span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -186,7 +186,7 @@
{{else}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
class='ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "STOPPED" "SUCCEEDED" "CREATE_FAILED"}}disabled {{else}} blue {{end}}button'
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .JobType "INFERENCE"}}modelarts/inference-job{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}modelarts/train-job{{else}}cloudbrain/train-job{{end}}{{end}}'
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}/{{if eq .JobType "INFERENCE"}}modelarts/inference-job{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}modelarts/train-job{{else if eq .Cloudbrain.Type 0}}cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}grampus/train-job{{end}}{{end}}'
data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
{{$.i18n.Tr "repo.stop"}}
</a>
@@ -203,7 +203,7 @@
{{end}}
<!-- 删除任务 -->
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"


+ 7
- 3
templates/user/dashboard/feeds.tmpl View File

@@ -86,6 +86,10 @@
{{$.i18n.Tr "action.task_createmodel" .GetRepoLink .RefName .RefName | Str2html}}
{{else if eq .GetOpType 31}}
{{$.i18n.Tr "action.task_gputrainjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 32}}
{{$.i18n.Tr "action.task_c2netnputrainjob" .GetRepoLink .Content .RefName | Str2html}}
{{else if eq .GetOpType 33}}
{{$.i18n.Tr "action.task_c2netgputrainjob" .GetRepoLink .Content .RefName | Str2html}}
{{end}}
</p>
{{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
@@ -107,11 +111,11 @@
<span class="text truncate issue title">{{index .GetIssueInfos 1 | RenderEmoji}}</span>
{{else if or (eq .GetOpType 10) (eq .GetOpType 21) (eq .GetOpType 22) (eq .GetOpType 23)}}
<a href="{{.GetCommentLink}}" class="text truncate issue title">{{.GetIssueTitle | RenderEmoji}}</a>
<p class="text light grey">{{index .GetIssueInfos 1 | RenderEmoji}}</p>
<p class="text truncate pre-wrap light grey">{{index .GetIssueInfos 1 | RenderEmoji}}</p>
{{else if eq .GetOpType 11}}
<p class="text light grey">{{index .GetIssueInfos 1}}</p>
<p class="text truncate pre-wrap light grey">{{index .GetIssueInfos 1}}</p>
{{else if or (eq .GetOpType 12) (eq .GetOpType 13) (eq .GetOpType 14) (eq .GetOpType 15)}}
<span class="text truncate issue title">{{.GetIssueTitle | RenderEmoji}}</span>
<span class="text truncate pre-wrap issue title">{{.GetIssueTitle | RenderEmoji}}</span>
{{end}}
<p class="text italic light grey">{{TimeSince .GetCreate $.i18n.Lang}}</p>
</div>


+ 1155
- 0
web_src/js/components/dataset/selectDataset.vue
File diff suppressed because it is too large
View File


+ 1
- 1
web_src/js/components/images/Images.vue View File

@@ -672,7 +672,7 @@ export default {
},
filters:{
transformType(val){
if(val==0){
if(val==0 || val==2){
return "GPU"
}
},


+ 292
- 0
web_src/js/components/images/selectGrampusImages.vue View File

@@ -0,0 +1,292 @@
<template>
<div
class="inline required field"
:class="{ unite: benchmarkNew, min_title: benchmarkNew }"
>
<label
v-if="benchmarkNew"
class="label-fix-width"
style="font-weight: normal"
>镜像</label
>
<label v-else>镜像</label>
<input
v-if="benchmarkNew"
type="text"
name="image"
:value="imageAddress"
style="width: 48.5%"
placeholder="选择镜像或输入镜像地址"
/>
<input
v-else
type="text"
name="image"
:value="imageAddress"
placeholder="选择镜像或输入镜像地址"
/>
<el-button
type="text"
@click="dialogVisible = true"
icon="el-icon-plus"
style="color: #0366d6"
>选择镜像
</el-button>
<el-dialog title="选择镜像" :visible.sync="dialogVisible" width="50%">
<div
class="ui icon input"
style="z-index: 9999; position: absolute; right: 50px; height: 30px"
>
<i
class="search icon"
style="cursor: pointer; pointer-events: auto"
></i>
<input
type="text"
placeholder="搜镜像Tag/描述/标签..."
v-model="search"
/>
</div>
<el-tabs v-model="activeName" @tab-click="handleClick">
<el-tab-pane label="公开镜像" name="first" v-loading="loadingPublic">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 0;
border-bottom: 1px solid #f5f5f5;
"
v-for="(publicData, index) in tableDataPublic"
:key="index"
>
<div style="width: 90%">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
"
>
<div style="display: flex; align-items: center">
<span
class="panel_dataset_name text-over"
style="margin-left: 0"
>{{ publicData.tag }}
</span>
<img
v-if="publicData.type == 5"
src="/img/jian.svg"
style="margin-left: 0.5rem"
/>
</div>

<div v-if="!!publicData.topics" class="text-over">
<span
v-for="(topic, index) in publicData.topics"
class="ui repo-topic label topic"
>{{ topic }}</span
>
</div>
</div>
<div style="margin-top: 8px; display: flex">
<a
v-if="publicData.relAvatarLink || publicData.userName"
:title="publicData.userName"
style="cursor: default"
>
<img
class="ui avatar mini image"
style="width: 20px; height: 20px"
:src="publicData.relAvatarLink"
/>
</a>
<a v-else
><img
class="ui avatar mini image"
title="Ghost"
src="/user/avatar/ghost/-1"
style="width: 20px; height: 20px"
/></a>
<span class="panel_datset_desc">{{
publicData.description
}}</span>
</div>
</div>
<div>
<button
class="ui primary basic button mini"
@click.stop.prevent="
selectImages(publicData.place, publicData.tag)
"
>
使用
</button>
</div>
</div>
<div
class="ui container"
style="margin-top: 50px; text-align: center"
>
<el-pagination
background
@current-change="handleCurrentChangePublic"
:current-page="currentPagePublic"
:page-size="pageSizePublic"
layout="total, prev, pager, next"
:total="totalNumPublic"
>
</el-pagination>
</div>
</el-tab-pane>
</el-tabs>
</el-dialog>
</div>
</template>

<script>
const { _AppSubUrl, _StaticUrlPrefix, csrf } = window.config;

export default {
components: {},
data() {
return {
dialogVisible: false,
benchmarkNew: false,
imageAddress: "",
activeName: "first",
search: "",
checked: false,
currentPagePublic: 1,
pageSizePublic: 5,
totalNumPublic: 0,
paramsPublic: {
page: 1,
pageSize: 5,
q: "",
recommend: false,
cloudbrainType: 2,
},
tableDataPublic: [],
loadingPublic: false,
};
},
methods: {
handleClick(tab, event) {
this.search = "";
if (tab.name == "first") {
this.paramsPublic.q = "";
this.getImageListPublic();
}
},
tableHeaderStyle({ row, column, rowIndex, columnIndex }) {
if (rowIndex === 0) {
return "background:#f5f5f6;color:#606266";
}
},

handleCurrentChangePublic(val) {
this.paramsPublic.page = val;
this.getImageListPublic();
},

getImageListPublic() {
this.loadingPublic = true;
this.$axios
.get("/explore/images/public", {
params: this.paramsPublic,
})
.then((res) => {
this.totalNumPublic = res.data.count;
this.tableDataPublic = res.data.images;
this.loadingPublic = false;
});
},

searchName() {
if (this.activeName == "first") {
this.paramsPublic.q = this.search;
this.paramsPublic.page = 1;
this.getImageListPublic();
}
},

selectImages(place) {
this.imageAddress = place;
this.dialogVisible = false;
},
},
watch: {
search(val) {
if (this.activeName == "first") {
this.paramsPublic.q = val;
this.getImageListPublic();
}
},
},
mounted() {
this.getImageListPublic();
if (
location.href.indexOf("benchmark") !== -1 ||
location.href.indexOf("train-job") !== -1
) {
this.benchmarkNew = true;
}
},
created() {},
};
</script>

<style scoped>
.header-wrapper {
background-color: #f5f5f6;
padding-top: 15px;
}

.image_text {
padding: 25px 0 55px 0;
}

#header {
position: relative;
top: -40px;
}

#success {
background-color: #5bb973;
color: white;
}

.text-over {
overflow: hidden;
text-overflow: ellipsis;
vertical-align: middle;
white-space: nowrap;
}

.image_title {
display: inline-block;
width: 80%;
cursor: default;
color: rgb(66, 98, 144);
}

.image_desc {
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
display: -webkit-box;
text-overflow: ellipsis;
overflow: hidden;
}

.heart-stroke {
stroke: #666;
stroke-width: 2;
fill: #fff;
}

.stars_active {
fill: #fa8c16 !important;
stroke: #fa8c16 !important;
}
</style>

+ 560
- 367
web_src/js/components/images/selectImages.vue View File

@@ -1,376 +1,569 @@
<template>

<div class="inline required field" :class="{ 'unite': benchmarkNew, 'min_title': benchmarkNew}">
<label v-if="benchmarkNew" style="font-weight: normal;">镜像</label>
<label v-else>镜像</label>
<span v-if="benchmarkNew">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span>
<input v-if="benchmarkNew" type="text" name="image" :value="imageAddress" style="width: 48.5%;"
placeholder="选择镜像或输入镜像地址" required>
<input v-else type="text" name="image" :value="imageAddress" placeholder="选择镜像或输入镜像地址" required>
<el-button type="text" @click="dialogVisible = true" icon="el-icon-plus" style="color: #0366d6;">选择镜像
</el-button>
<el-dialog title="选择镜像" :visible.sync="dialogVisible" width="50%">
<div class="ui icon input" style="z-index: 9999;position: absolute;right: 50px;height:30px;">
<i class="search icon" style="cursor: pointer;pointer-events:auto"></i>
<input type="text" placeholder="搜镜像Tag/描述/标签..." v-model="search">
<div
class="inline required field"
:class="{ min_title: benchmarkNew, unite: benchmark }"
>
<label
v-if="benchmarkNew"
class="label-fix-width"
style="font-weight: normal"
>镜像</label
>
<label v-else>镜像</label>
<span v-if="benchmark">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</span>
<input
v-if="benchmarkNew"
type="text"
name="image"
:value="imageAddress"
style="width: 48.5%"
placeholder="选择镜像或输入镜像地址"
required
/>
<input
v-else
type="text"
name="image"
:value="imageAddress"
placeholder="选择镜像或输入镜像地址"
required
/>
<el-button
type="text"
@click="dialogVisible = true"
icon="el-icon-plus"
style="color: #0366d6"
>选择镜像
</el-button>
<el-dialog title="选择镜像" :visible.sync="dialogVisible" width="50%">
<div
class="ui icon input"
style="z-index: 9999; position: absolute; right: 50px; height: 30px"
>
<i
class="search icon"
style="cursor: pointer; pointer-events: auto"
></i>
<input
type="text"
placeholder="搜镜像Tag/描述/标签..."
v-model="search"
/>
</div>
<el-tabs v-model="activeName" @tab-click="handleClick">
<el-tab-pane label="公开镜像" name="first" v-loading="loadingPublic">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 0;
border-bottom: 1px solid #f5f5f5;
"
v-for="(publicData, index) in tableDataPublic"
:key="index"
>
<div style="width: 90%">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
"
>
<div style="display: flex; align-items: center">
<span
class="panel_dataset_name text-over"
style="margin-left: 0"
>{{ publicData.tag }}
</span>
<img
v-if="publicData.type == 5"
src="/img/jian.svg"
style="margin-left: 0.5rem"
/>
</div>

<div v-if="!!publicData.topics" class="text-over">
<span
v-for="(topic, index) in publicData.topics"
class="ui repo-topic label topic"
>{{ topic }}</span
>
</div>
</div>
<div style="margin-top: 8px; display: flex">
<a
v-if="publicData.relAvatarLink || publicData.userName"
:title="publicData.userName"
style="cursor: default"
>
<img
class="ui avatar mini image"
style="width: 20px; height: 20px"
:src="publicData.relAvatarLink"
/>
</a>
<a v-else
><img
class="ui avatar mini image"
title="Ghost"
src="/user/avatar/ghost/-1"
style="width: 20px; height: 20px"
/></a>
<span class="panel_datset_desc">{{
publicData.description
}}</span>
</div>
</div>
<el-tabs v-model="activeName" @tab-click="handleClick">
<el-tab-pane label="公开镜像" name="first" v-loading="loadingPublic">
<div style="display: flex;align-items: center;justify-content: space-between;padding: 1rem 0;border-bottom:1px solid #F5F5F5"
v-for="(publicData,index) in tableDataPublic" :key="index">
<div style="width: 90%;">
<div style="display: flex;align-items: center;justify-content: space-between;">
<div style="display: flex;align-items: center;">
<span class="panel_dataset_name text-over"
style="margin-left: 0;">{{publicData.tag}} </span>
<img v-if="publicData.type==5" src="/img/jian.svg" style="margin-left: 0.5rem;">
</div>

<div v-if="!!publicData.topics" class="text-over">
<span v-for="(topic,index) in publicData.topics"
class="ui repo-topic label topic">{{topic}}</span>
</div>
</div>
<div style="margin-top: 8px;display: flex;">
<a v-if="publicData.relAvatarLink||publicData.userName" :title="publicData.userName"
style="cursor: default;">
<img class="ui avatar mini image" style="width: 20px;height: 20px;"
:src="publicData.relAvatarLink">
</a>
<a v-else><img class="ui avatar mini image" title="Ghost" src="/user/avatar/ghost/-1"
style="width: 20px;height: 20px;"></a>
<span class="panel_datset_desc">{{publicData.description}}</span>
</div>
</div>
<div>
<button class="ui primary basic button mini"
@click.stop.prevent="selectImages(publicData.place,publicData.tag)">使用</button>
</div>
</div>
<div class="ui container" style="margin-top:50px;text-align:center">
<el-pagination background @current-change="handleCurrentChangePublic"
:current-page="currentPagePublic" :page-size="pageSizePublic"
layout="total, prev, pager, next" :total="totalNumPublic">
</el-pagination>
</div>
</el-tab-pane>

<el-tab-pane label="我的镜像" name="second" v-loading="loadingCustom">
<div style="display: flex;align-items: center;justify-content: space-between;padding: 1rem 0;border-bottom:1px solid #F5F5F5"
v-for="(customData,index) in tableDataCustom" :key="index">
<div style="width: 90%;">
<div style="display: flex;align-items: center;justify-content: space-between;">
<span class="panel_dataset_name text-over" style="margin-left: 0;">{{customData.tag}}
</span>
<div v-if="!!customData.topics" class="text-over">
<span v-for="(topic,index) in customData.topics"
class="ui repo-topic label topic">{{topic}}</span>
</div>
</div>
<div style="margin-top: 8px;display: flex;">
<a v-if="customData.relAvatarLink||customData.userName" :title="customData.userName"
style="cursor: default;">
<img class="ui avatar mini image" style="width: 20px;height: 20px;"
:src="customData.relAvatarLink">
</a>
<a v-else><img class="ui avatar mini image" title="Ghost" src="/user/avatar/ghost/-1"
style="width: 20px;height: 20px;"></a>

<span class="panel_datset_desc">{{customData.description}}</span>
</div>
</div>
<div>
<button v-if="customData.status===1" class="ui primary basic button mini"
@click.stop.prevent="selectImages(customData.place,customData.tag)">使用</button>
<span v-if="customData.status===0" style="display: flex;align-items: center;">
<i class="CREATING"></i>
<span style="margin-left: 0.4em;font-size: 12px;color: #5A5A5A;">提交中</span>
</span>
<span v-if="customData.status===2" style="display: flex;align-items: center;">
<i class="FAILED"></i>
<el-tooltip class="item" effect="dark" content="检测提交镜像是否大小超过20G!" placement="left">
<span style="margin-left: 0.4em;font-size: 12px;color:red;">提交失败</span>
</el-tooltip>

</span>
</div>
</div>
<div class="ui container" style="margin-top:50px;text-align:center">
<el-pagination background @current-change="handleCurrentChangeCustom"
:current-page="currentPageCustom" :page-size="pageSizeCustom"
layout="total, prev, pager, next" :total="totalNumCustom">
</el-pagination>
</div>
</el-tab-pane>

<el-tab-pane label="我收藏的镜像" name="third">
<div style="display: flex;align-items: center;justify-content: space-between;padding: 1rem 0;border-bottom:1px solid #F5F5F5"
v-for="(starData,index) in tableDataStar" :key="index">
<div style="width: 90%;">
<div style="display: flex;align-items: center;justify-content: space-between;">
<div style="display: flex;align-items: center;">
<span class="panel_dataset_name text-over" style="margin-left: 0;">{{starData.tag}}
</span>
<img v-if="starData.type==5" src="/img/jian.svg" style="margin-left: 0.5rem;">
</div>

<div v-if="!!starData.topics" class="text-over">
<span v-for="(topic,index) in starData.topics"
class="ui repo-topic label topic">{{topic}}</span>
</div>
</div>
<div style="margin-top: 8px;display: flex;">
<a v-if="starData.relAvatarLink||starData.userName" :title="starData.userName"
style="cursor: default;">
<img class="ui avatar mini image" style="width: 20px;height: 20px;"
:src="starData.relAvatarLink">
</a>
<a v-else><img class="ui avatar mini image" title="Ghost" src="/user/avatar/ghost/-1"
style="width: 20px;height: 20px;"></a>
<span class="panel_datset_desc">{{starData.description}}</span>
</div>
</div>
<div>
<button class="ui primary basic button mini"
@click.stop.prevent="selectImages(starData.place,starData.tag)">使用</button>
</div>
</div>
<div class="ui container" style="margin-top:50px;text-align:center">
<el-pagination background @current-change="handleCurrentChangeStar"
:current-page="currentPageStar" :page-size="pageSizeStar" layout="total, prev, pager, next"
:total="totalNumStar">
</el-pagination>
</div>
</el-tab-pane>
</el-tabs>
</el-dialog>
</div>



<div>
<button
class="ui primary basic button mini"
@click.stop.prevent="
selectImages(publicData.place, publicData.tag)
"
>
使用
</button>
</div>
</div>
<div
class="ui container"
style="margin-top: 50px; text-align: center"
>
<el-pagination
background
@current-change="handleCurrentChangePublic"
:current-page="currentPagePublic"
:page-size="pageSizePublic"
layout="total, prev, pager, next"
:total="totalNumPublic"
>
</el-pagination>
</div>
</el-tab-pane>

<el-tab-pane label="我的镜像" name="second" v-loading="loadingCustom">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 0;
border-bottom: 1px solid #f5f5f5;
"
v-for="(customData, index) in tableDataCustom"
:key="index"
>
<div style="width: 90%">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
"
>
<span
class="panel_dataset_name text-over"
style="margin-left: 0"
>{{ customData.tag }}
</span>
<div v-if="!!customData.topics" class="text-over">
<span
v-for="(topic, index) in customData.topics"
class="ui repo-topic label topic"
>{{ topic }}</span
>
</div>
</div>
<div style="margin-top: 8px; display: flex">
<a
v-if="customData.relAvatarLink || customData.userName"
:title="customData.userName"
style="cursor: default"
>
<img
class="ui avatar mini image"
style="width: 20px; height: 20px"
:src="customData.relAvatarLink"
/>
</a>
<a v-else
><img
class="ui avatar mini image"
title="Ghost"
src="/user/avatar/ghost/-1"
style="width: 20px; height: 20px"
/></a>

<span class="panel_datset_desc">{{
customData.description
}}</span>
</div>
</div>
<div>
<button
v-if="customData.status === 1"
class="ui primary basic button mini"
@click.stop.prevent="
selectImages(customData.place, customData.tag)
"
>
使用
</button>
<span
v-if="customData.status === 0"
style="display: flex; align-items: center"
>
<i class="CREATING"></i>
<span
style="margin-left: 0.4em; font-size: 12px; color: #5a5a5a"
>提交中</span
>
</span>
<span
v-if="customData.status === 2"
style="display: flex; align-items: center"
>
<i class="FAILED"></i>
<el-tooltip
class="item"
effect="dark"
content="检测提交镜像是否大小超过20G!"
placement="left"
>
<span style="margin-left: 0.4em; font-size: 12px; color: red"
>提交失败</span
>
</el-tooltip>
</span>
</div>
</div>
<div
class="ui container"
style="margin-top: 50px; text-align: center"
>
<el-pagination
background
@current-change="handleCurrentChangeCustom"
:current-page="currentPageCustom"
:page-size="pageSizeCustom"
layout="total, prev, pager, next"
:total="totalNumCustom"
>
</el-pagination>
</div>
</el-tab-pane>

<el-tab-pane label="我收藏的镜像" name="third">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 0;
border-bottom: 1px solid #f5f5f5;
"
v-for="(starData, index) in tableDataStar"
:key="index"
>
<div style="width: 90%">
<div
style="
display: flex;
align-items: center;
justify-content: space-between;
"
>
<div style="display: flex; align-items: center">
<span
class="panel_dataset_name text-over"
style="margin-left: 0"
>{{ starData.tag }}
</span>
<img
v-if="starData.type == 5"
src="/img/jian.svg"
style="margin-left: 0.5rem"
/>
</div>

<div v-if="!!starData.topics" class="text-over">
<span
v-for="(topic, index) in starData.topics"
class="ui repo-topic label topic"
>{{ topic }}</span
>
</div>
</div>
<div style="margin-top: 8px; display: flex">
<a
v-if="starData.relAvatarLink || starData.userName"
:title="starData.userName"
style="cursor: default"
>
<img
class="ui avatar mini image"
style="width: 20px; height: 20px"
:src="starData.relAvatarLink"
/>
</a>
<a v-else
><img
class="ui avatar mini image"
title="Ghost"
src="/user/avatar/ghost/-1"
style="width: 20px; height: 20px"
/></a>
<span class="panel_datset_desc">{{
starData.description
}}</span>
</div>
</div>
<div>
<button
class="ui primary basic button mini"
@click.stop.prevent="selectImages(starData.place, starData.tag)"
>
使用
</button>
</div>
</div>
<div
class="ui container"
style="margin-top: 50px; text-align: center"
>
<el-pagination
background
@current-change="handleCurrentChangeStar"
:current-page="currentPageStar"
:page-size="pageSizeStar"
layout="total, prev, pager, next"
:total="totalNumStar"
>
</el-pagination>
</div>
</el-tab-pane>
</el-tabs>
</el-dialog>
</div>
</template>

<script>

const { _AppSubUrl, _StaticUrlPrefix, csrf } = window.config;




export default {
components: {

},
data() {
return {
dialogVisible: false,
benchmarkNew: false,
imageAddress: '',
activeName: 'first',
search: '',
checked: false,
currentPagePublic: 1,
pageSizePublic: 5,
totalNumPublic: 0,
paramsPublic: { page: 1, pageSize: 5, q: '', recommend: false },
tableDataPublic: [],
loadingPublic: false,

currentPageCustom: 1,
pageSizeCustom: 5,
totalNumCustom: 0,
paramsCustom: { page: 1, pageSize: 5, q: '' },
tableDataCustom: [],
starCustom: [],
loadingCustom: false,

currentPageStar: 1,
pageSizeStar: 5,
totalNumStar: 0,
paramsStar: { page: 1, pageSize: 5, q: '' },
tableDataStar: [],
loadingStar: false
};
},
methods: {
handleClick(tab, event) {
this.search = ''
if (tab.name == "first") {
this.paramsPublic.q = ''
this.getImageListPublic()
}
if (tab.name == "second") {
this.getImageListCustom()
}
if (tab.name == "third") {
this.getImageListStar()
}

},
tableHeaderStyle({ row, column, rowIndex, columnIndex }) {

if (rowIndex === 0) {
return 'background:#f5f5f6;color:#606266'
}

},

handleCurrentChangePublic(val) {
this.paramsPublic.page = val
this.getImageListPublic()

},

handleCurrentChangeCustom(val) {
this.paramsCustom.page = val
this.getImageListCustom()

},
handleCurrentChangeStar(val) {
this.paramsStar.page = val
this.getImageListStar()

},
getImageListPublic() {
this.loadingPublic = true
this.$axios.get('/explore/images/public', {
params: this.paramsPublic
}).then((res) => {
this.totalNumPublic = res.data.count
this.tableDataPublic = res.data.images
this.loadingPublic = false
})
},

getImageListCustom() {
this.loadingCustom = true
this.$axios.get('/explore/images/custom', {
params: this.paramsCustom
}).then((res) => {
this.totalNumCustom = res.data.count
this.tableDataCustom = res.data.images
this.tableDataCustom.forEach(element => {
this.starCustom.push({ id: element.id, })

});
this.loadingCustom = false
})
},

getImageListStar() {
this.loadingStar = true
this.$axios.get('/explore/images/star', {
params: this.paramsStar
}).then((res) => {
this.totalNumStar = res.data.count
this.tableDataStar = res.data.images
this.loadingStar = false
})
},
searchName() {
if (this.activeName == 'first') {
this.paramsPublic.q = this.search
this.paramsPublic.page = 1
this.getImageListPublic()
}
if (this.activeName == 'second') {
this.paramsCustom.q = this.search
this.paramsCustom.page = 1
this.getImageListCustom()
}
if (this.activeName == 'third') {
this.paramsStar.q = this.search
this.paramsStar.page = 1
this.getImageListStar()
}

},
selectImages(place) {
this.imageAddress = place
this.dialogVisible = false
},

},
watch: {
search(val) {
if (this.activeName == 'first') {
this.paramsPublic.q = val
this.getImageListPublic()
}
if (this.activeName == 'second') {
this.paramsCustom.q = val
this.getImageListCustom()
}
if (this.activeName == 'third') {
this.paramsStar.q = val
this.getImageListStar()
}
}

},
mounted() {
this.getImageListPublic()
if (location.href.indexOf('benchmark') !== -1 || location.href.indexOf('train-job') !== -1) {
this.benchmarkNew = true
}
},
created() {

}

const { _AppSubUrl, _StaticUrlPrefix, csrf } = window.config;

export default {
components: {},
data() {
return {
dialogVisible: false,
benchmarkNew: false,
benchmark: false,
imageAddress: "",
activeName: "first",
search: "",
checked: false,
currentPagePublic: 1,
pageSizePublic: 5,
totalNumPublic: 0,
paramsPublic: { page: 1, pageSize: 5, q: "", recommend: false },
tableDataPublic: [],
loadingPublic: false,

currentPageCustom: 1,
pageSizeCustom: 5,
totalNumCustom: 0,
paramsCustom: { page: 1, pageSize: 5, q: "" },
tableDataCustom: [],
starCustom: [],
loadingCustom: false,

currentPageStar: 1,
pageSizeStar: 5,
totalNumStar: 0,
paramsStar: { page: 1, pageSize: 5, q: "" },
tableDataStar: [],
loadingStar: false,
};
</script>

<style scoped>
.header-wrapper {
background-color: #f5f5f6;
padding-top: 15px;
},
methods: {
handleClick(tab, event) {
this.search = "";
if (tab.name == "first") {
this.paramsPublic.q = "";
this.getImageListPublic();
}
if (tab.name == "second") {
this.getImageListCustom();
}
if (tab.name == "third") {
this.getImageListStar();
}
},
tableHeaderStyle({ row, column, rowIndex, columnIndex }) {
if (rowIndex === 0) {
return "background:#f5f5f6;color:#606266";
}
},

handleCurrentChangePublic(val) {
this.paramsPublic.page = val;
this.getImageListPublic();
},

handleCurrentChangeCustom(val) {
this.paramsCustom.page = val;
this.getImageListCustom();
},
handleCurrentChangeStar(val) {
this.paramsStar.page = val;
this.getImageListStar();
},
getImageListPublic() {
this.loadingPublic = true;
this.$axios
.get("/explore/images/public", {
params: this.paramsPublic,
})
.then((res) => {
this.totalNumPublic = res.data.count;
this.tableDataPublic = res.data.images;
this.loadingPublic = false;
});
},

getImageListCustom() {
this.loadingCustom = true;
this.$axios
.get("/explore/images/custom", {
params: this.paramsCustom,
})
.then((res) => {
this.totalNumCustom = res.data.count;
this.tableDataCustom = res.data.images;
this.tableDataCustom.forEach((element) => {
this.starCustom.push({ id: element.id });
});
this.loadingCustom = false;
});
},

getImageListStar() {
this.loadingStar = true;
this.$axios
.get("/explore/images/star", {
params: this.paramsStar,
})
.then((res) => {
this.totalNumStar = res.data.count;
this.tableDataStar = res.data.images;
this.loadingStar = false;
});
},
searchName() {
if (this.activeName == "first") {
this.paramsPublic.q = this.search;
this.paramsPublic.page = 1;
this.getImageListPublic();
}
if (this.activeName == "second") {
this.paramsCustom.q = this.search;
this.paramsCustom.page = 1;
this.getImageListCustom();
}
if (this.activeName == "third") {
this.paramsStar.q = this.search;
this.paramsStar.page = 1;
this.getImageListStar();
}
},
selectImages(place) {
this.imageAddress = place;
this.dialogVisible = false;
},
},
watch: {
search(val) {
if (this.activeName == "first") {
this.paramsPublic.q = val;
this.getImageListPublic();
}
if (this.activeName == "second") {
this.paramsCustom.q = val;
this.getImageListCustom();
}
if (this.activeName == "third") {
this.paramsStar.q = val;
this.getImageListStar();
}
},
},
mounted() {
this.getImageListPublic();
if (
location.href.indexOf("train-job") !== -1 ||
location.href.indexOf("benchmark") !== -1
) {
this.benchmarkNew = true;
}

.image_text {
padding: 25px 0 55px 0;
}

#header {
position: relative;
top: -40px;
}

#success {
background-color: #5bb973;
color: white;
}

.text-over {
overflow: hidden;
text-overflow: ellipsis;
vertical-align: middle;
white-space: nowrap;
}

.image_title {
display: inline-block;
width: 80%;
cursor: default;
color: rgb(66, 98, 144);
}

.image_desc {
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
display: -webkit-box;
text-overflow: ellipsis;
overflow: hidden;
}

.heart-stroke {
stroke: #666;
stroke-width: 2;
fill: #fff
if (location.href.indexOf("cloudbrain/benchmark/") !== -1) {
this.benchmark = true;
}
},
created() {},
};
</script>

.stars_active {
fill: #FA8C16 !important;
stroke: #FA8C16 !important
}
</style>
<style scoped>
.header-wrapper {
background-color: #f5f5f6;
padding-top: 15px;
}

.image_text {
padding: 25px 0 55px 0;
}

#header {
position: relative;
top: -40px;
}

#success {
background-color: #5bb973;
color: white;
}

.text-over {
overflow: hidden;
text-overflow: ellipsis;
vertical-align: middle;
white-space: nowrap;
}

.image_title {
display: inline-block;
width: 80%;
cursor: default;
color: rgb(66, 98, 144);
}

.image_desc {
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
display: -webkit-box;
text-overflow: ellipsis;
overflow: hidden;
}

.heart-stroke {
stroke: #666;
stroke-width: 2;
fill: #fff;
}

.stars_active {
fill: #fa8c16 !important;
stroke: #fa8c16 !important;
}
</style>

+ 13
- 0
web_src/js/features/images.js View File

@@ -1,6 +1,7 @@
import Images from '../components/images/Images.vue';
import adminImages from '../components/images/adminImages.vue';
import selectImages from '../components/images/selectImages.vue';
import selectGrampusImages from '../components/images/selectGrampusImages.vue';
import Vue from 'vue';
export default async function initImage(){
function validate() {
@@ -209,7 +210,19 @@ export default async function initImage(){
render: h => h(selectImages)
});
}
function initVueselectGrampusImages() {
const el = document.getElementById('images-new-grampus');
if (!el) {
return;
}
new Vue({
el:el,
render: h => h(selectGrampusImages)
});
}
initVueImages()
initVueAdminImages()
initVueselectImages()
initVueselectGrampusImages()
}

+ 91
- 65
web_src/js/features/letteravatar.js View File

@@ -1,74 +1,100 @@
/**
* LetterAvatar
*
*
* Artur Heinze
* Create Letter avatar based on Initials
* based on https://gist.github.com/leecrossley/6027780
*/
(function(w, d){
function LetterAvatar (name, size, color) {
name = name || '';
size = size || 60;
var colours = [
"#1abc9c", "#2ecc71", "#3498db", "#9b59b6", "#34495e", "#16a085", "#27ae60", "#2980b9", "#8e44ad", "#2c3e50",
"#f1c40f", "#e67e22", "#e74c3c", "#00bcd4", "#95a5a6", "#f39c12", "#d35400", "#c0392b", "#bdc3c7", "#7f8c8d"
],
nameSplit = String(name).split(' '),
initials, charIndex, colourIndex, canvas, context, dataURI;
if (nameSplit.length == 1) {
initials = nameSplit[0] ? nameSplit[0].charAt(0):'?';
} else {
initials = nameSplit[0].charAt(0) + nameSplit[1].charAt(0);
}
if (w.devicePixelRatio) {
size = (size * w.devicePixelRatio);
}
charIndex = (initials == '?' ? 72 : initials.charCodeAt(0)) - 64;
colourIndex = charIndex % 20;
canvas = d.createElement('canvas');
canvas.width = size;
canvas.height = size;
context = canvas.getContext("2d");
context.fillStyle = color ? color : colours[colourIndex - 1];
context.fillRect (0, 0, canvas.width, canvas.height);
context.font = Math.round(canvas.width/2)+"px 'Microsoft Yahei'";
context.textAlign = "center";
context.fillStyle = "#FFF";
context.fillText(initials, size / 2, size / 1.5);
dataURI = canvas.toDataURL();
canvas = null;
return dataURI;
(function (w, d) {
function LetterAvatar(name, size, color) {
name = name || "";
size = size || 60;
var colours = [
"#1abc9c",
"#2ecc71",
"#3498db",
"#9b59b6",
"#34495e",
"#16a085",
"#27ae60",
"#2980b9",
"#8e44ad",
"#2c3e50",
"#f1c40f",
"#e67e22",
"#e74c3c",
"#00bcd4",
"#95a5a6",
"#f39c12",
"#d35400",
"#c0392b",
"#bdc3c7",
"#7f8c8d",
],
nameSplit = String(name).split(" "),
initials,
charIndex,
colourIndex,
canvas,
context,
dataURI;
if (nameSplit.length == 1) {
initials = nameSplit[0] ? nameSplit[0].charAt(0) : "?";
} else {
initials = nameSplit[0].charAt(0) + nameSplit[1].charAt(0);
}
let initials1 = initials.toUpperCase();
if (w.devicePixelRatio) {
size = size * w.devicePixelRatio;
}
LetterAvatar.transform = function() {
Array.prototype.forEach.call(d.querySelectorAll('img[avatar]'), function(img, name, color) {
name = img.getAttribute('avatar');
color = img.getAttribute('color');
img.src = LetterAvatar(name, img.getAttribute('width'), color);
img.removeAttribute('avatar');
img.setAttribute('alt', name);
});
};
// AMD support
if (typeof define === 'function' && define.amd) {
define(function () { return LetterAvatar; });

charIndex = (initials == "?" ? 72 : initials.charCodeAt(0)) - 64;
colourIndex = charIndex % 20;
canvas = d.createElement("canvas");
canvas.width = size;
canvas.height = size;
context = canvas.getContext("2d");

context.fillStyle = color ? color : colours[colourIndex - 1];
context.fillRect(0, 0, canvas.width, canvas.height);
context.font = Math.round(canvas.width / 2) + "px 'Microsoft Yahei'";
context.textAlign = "center";
context.fillStyle = "#FFF";
context.fillText(initials1, size / 2, size / 1.5);
dataURI = canvas.toDataURL();
canvas = null;
return dataURI;
}
LetterAvatar.transform = function () {
Array.prototype.forEach.call(
d.querySelectorAll("img[avatar]"),
function (img, name, color) {
name = img.getAttribute("avatar");
color = img.getAttribute("color");
img.src = LetterAvatar(name, img.getAttribute("width"), color);
img.removeAttribute("avatar");
img.setAttribute("alt", name);
}
);
};
// AMD support
if (typeof define === "function" && define.amd) {
define(function () {
return LetterAvatar;
});

// CommonJS and Node.js module support.
} else if (typeof exports !== 'undefined') {
// Support Node.js specific `module.exports` (which can be a function)
if (typeof module != 'undefined' && module.exports) {
exports = module.exports = LetterAvatar;
}
// But always support CommonJS module 1.1.1 spec (`exports` cannot be a function)
exports.LetterAvatar = LetterAvatar;
} else {
window.LetterAvatar = LetterAvatar;
d.addEventListener('DOMContentLoaded', function(event) {
LetterAvatar.transform();
});
} else if (typeof exports !== "undefined") {
// Support Node.js specific `module.exports` (which can be a function)
if (typeof module != "undefined" && module.exports) {
exports = module.exports = LetterAvatar;
}
})(window, document);
// But always support CommonJS module 1.1.1 spec (`exports` cannot be a function)
exports.LetterAvatar = LetterAvatar;
} else {
window.LetterAvatar = LetterAvatar;
d.addEventListener("DOMContentLoaded", function (event) {
LetterAvatar.transform();
});
}
})(window, document);

+ 2117
- 2142
web_src/js/index.js
File diff suppressed because it is too large
View File


+ 3
- 0
web_src/less/_base.less View File

@@ -489,6 +489,9 @@ code,
white-space: nowrap;
display: inline-block;
}
&.pre-wrap{
white-space:pre-wrap!important;
}

&.thin {
font-weight: normal;


+ 849
- 728
web_src/less/openi.less
File diff suppressed because it is too large
View File


Loading…
Cancel
Save