diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9fd00a763..8bc971f2c 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1096,9 +1096,48 @@ LOCATION = cn-south-222 BASE_PATH = attachment/ [modelarts] +ORGANIZATION = modelarts ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa PROJECT_NAME = cn-south-222_test USERNAME = test1 PASSWORD = Qizhi@test. DOMAIN = cn-south-222 + +[radar_map] +impact=0.3 +impact_watch=0.1 +impact_star=0.3 +impact_fork=0.3 +impact_code_download=0.2 +impact_comments=0.1 +impact_browser=0.1 + +completeness=0.1 +completeness_issues_closed=0.2 +completeness_releases=0.3 +completeness_develop_age=0.1 +completeness_dataset=0.1 +completeness_model=0.1 +completeness_wiki=0.1 + +liveness=0.3 +liveness_commit=0.2 +liveness_issue=0.2 +liveness_pr=0.2 +liveness_release=0.4 + +project_health=0.1 +project_health_issue_complete_ratio=100 + +team_health=0.1 +team_health_contributors=0.2 +team_health_key_contributors=0.6 +team_health_contributors_added=0.2 + +growth=0.1 +growth_code_lines=0.2 +growth_issue=0.2 +growth_contributors=0.2 +growth_commit=0.2 +growth_comments=0.2 diff --git a/models/attachment.go b/models/attachment.go index 684a38b21..d217a61a4 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -379,7 +379,7 @@ func GetUnDecompressAttachments() ([]*Attachment, error) { func getUnDecompressAttachments(e Engine) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) - return attachments, e.Where("decompress_state = ? and dataset_id != 0 and attachment.type = ? and (name like '%.zip' or name like '%.tar.gz' or name like '%.tgz')", DecompressStateInit, TypeCloudBrainOne).Find(&attachments) + return attachments, e.Where("decompress_state = ? and dataset_id != 0 and (name like '%.zip' or name like '%.tar.gz' or name like '%.tgz')", DecompressStateInit).Find(&attachments) } func GetAllPublicAttachments() ([]*AttachmentUsername, error) { @@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) { return total, nil } + +func GetAllAttachmentSize() (int64, error) { + return x.SumInt(&Attachment{}, "size") +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 4b2bec8e6..af5e9f169 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "time" + "xorm.io/builder" "xorm.io/xorm" @@ -27,6 +28,8 @@ const ( JobTypeDebug JobType = "DEBUG" JobTypeBenchmark JobType = "BENCHMARK" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" + JobTypeBrainScore JobType = "BRAINSCORE" + JobTypeTrain JobType = "TRAIN" ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 @@ -46,22 +49,29 @@ const ( ) type Cloudbrain struct { - ID int64 `xorm:"pk autoincr"` - JobID string `xorm:"INDEX NOT NULL"` - JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` - JobName string `xorm:"INDEX"` - Status string `xorm:"INDEX"` - UserID int64 `xorm:"INDEX"` - RepoID int64 `xorm:"INDEX"` - SubTaskName string `xorm:"INDEX"` - ContainerID string - ContainerIp string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - DeletedAt time.Time `xorm:"deleted"` - CanDebug bool `xorm:"-"` - CanDel bool `xorm:"-"` - Type int `xorm:"INDEX DEFAULT 0"` + ID int64 `xorm:"pk autoincr"` + JobID string `xorm:"INDEX NOT NULL"` + JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` + JobName string `xorm:"INDEX"` + Status string `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + SubTaskName string `xorm:"INDEX"` + ContainerID string + ContainerIp string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + Duration int64 `xorm:"INDEX duration"` + TrainJobDuration string + DeletedAt time.Time `xorm:"deleted"` + CanDebug bool `xorm:"-"` + CanDel bool `xorm:"-"` + Type int `xorm:"INDEX DEFAULT 0"` + + VersionID int64 `xorm:"INDEX DEFAULT 0"` + VersionName string + Uuid string + DatasetName string User *User `xorm:"-"` Repo *Repository `xorm:"-"` @@ -144,29 +154,49 @@ type CloudbrainsOptions struct { SortType string CloudbrainIDs []int64 // JobStatus CloudbrainStatus - Type int + Type int + JobType string } type TaskPod struct { TaskRoleStatus struct { Name string `json:"name"` } `json:"taskRoleStatus"` - TaskStatuses []struct { - TaskIndex int `json:"taskIndex"` - PodUID string `json:"podUid"` - PodIP string `json:"podIp"` - PodName string `json:"podName"` - ContainerID string `json:"containerId"` - ContainerIP string `json:"containerIp"` - ContainerGpus string `json:"containerGpus"` - State string `json:"state"` - StartAt time.Time `json:"startAt"` - FinishedAt time.Time `json:"finishedAt"` - ExitCode int `json:"exitCode"` - ExitDiagnostics string `json:"exitDiagnostics"` - RetriedCount int `json:"retriedCount"` - StartTime string - FinishedTime string - } `json:"taskStatuses"` + //TaskStatuses []struct { + // TaskIndex int `json:"taskIndex"` + // PodUID string `json:"podUid"` + // PodIP string `json:"podIp"` + // PodName string `json:"podName"` + // ContainerID string `json:"containerId"` + // ContainerIP string `json:"containerIp"` + // ContainerGpus string `json:"containerGpus"` + // State string `json:"state"` + // StartAt time.Time `json:"startAt"` + // FinishedAt time.Time `json:"finishedAt"` + // ExitCode int `json:"exitCode"` + // ExitDiagnostics string `json:"exitDiagnostics"` + // RetriedCount int `json:"retriedCount"` + // StartTime string + // FinishedTime string + //} `json:"taskStatuses"` + TaskStatuses []TaskStatuses `json:"taskStatuses"` +} + +type TaskStatuses struct { + TaskIndex int `json:"taskIndex"` + PodUID string `json:"podUid"` + PodIP string `json:"podIp"` + PodName string `json:"podName"` + ContainerID string `json:"containerId"` + ContainerIP string `json:"containerIp"` + ContainerGpus string `json:"containerGpus"` + State string `json:"state"` + StartAt time.Time `json:"startAt"` + FinishedAt time.Time `json:"finishedAt"` + ExitCode int `json:"exitCode"` + ExitDiagnostics string `json:"exitDiagnostics"` + RetriedCount int `json:"retriedCount"` + StartTime string + FinishedTime string } type TaskInfo struct { @@ -254,6 +284,11 @@ func ConvertToJobResultPayload(input map[string]interface{}) (JobResultPayload, err := json.Unmarshal(data, &jobResultPayload) jobResultPayload.JobStatus.StartTime = time.Unix(jobResultPayload.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05") jobResultPayload.JobStatus.EndTime = time.Unix(jobResultPayload.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05") + + if jobResultPayload.JobStatus.State == string(JobWaiting) { + jobResultPayload.JobStatus.StartTime = "-" + jobResultPayload.JobStatus.EndTime = "-" + } return jobResultPayload, err } @@ -530,6 +565,260 @@ type NotebookDelResult struct { InstanceID string `json:"instance_id"` } +type CreateTrainJobParams struct { + JobName string `json:"job_name"` + Description string `json:"job_desc"` + Config Config `json:"config"` + WorkspaceID string `json:"workspace_id"` +} + +type Config struct { + WorkServerNum int `json:"worker_server_num"` + AppUrl string `json:"app_url"` //训练作业的代码目录 + BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 + Parameter []Parameter `json:"parameter"` + DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL + //DatasetID string `json:"dataset_id"` + //DataVersionID string `json:"dataset_version_id"` + //DataSource []DataSource `json:"data_source"` + //SpecID int64 `json:"spec_id"` + EngineID int64 `json:"engine_id"` + //ModelID int64 `json:"model_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + //UserImageUrl string `json:"user_image_url"` + //UserCommand string `json:"user_command"` + CreateVersion bool `json:"create_version"` + //Volumes []Volumes `json:"volumes"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` +} + +type CreateConfigParams struct { + ConfigName string `json:"config_name"` + Description string `json:"config_desc"` + WorkServerNum int `json:"worker_server_num"` + AppUrl string `json:"app_url"` //训练作业的代码目录 + BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 + Parameter []Parameter `json:"parameter"` + DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL + //DatasetID string `json:"dataset_id"` + //DataVersionID string `json:"dataset_version_id"` + //DataSource []DataSource `json:"data_source"` + //SpecID int64 `json:"spec_id"` + EngineID int64 `json:"engine_id"` + //ModelID int64 `json:"model_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + //UserImageUrl string `json:"user_image_url"` + //UserCommand string `json:"user_command"` + //CreateVersion bool `json:"create_version"` + //Volumes []Volumes `json:"volumes"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` +} + +type Parameter struct { + Label string `json:"label"` + Value string `json:"value"` +} + +type Parameters struct { + Parameter []Parameter `json:"parameter"` +} + +type DataSource struct { + DatasetID string `json:"dataset_id"` + DatasetVersion string `json:"dataset_version"` + Type string `json:"type"` + DataUrl string `json:"data_url"` +} + +type Volumes struct { + Nfs Nfs `json:"nfs"` + HostPath HostPath `json:"host_path"` +} + +type Nfs struct { + ID string `json:"id"` + SourcePath string `json:"src_path"` + DestPath string `json:"dest_path"` + ReadOnly bool `json:"read_only"` +} + +type HostPath struct { + SourcePath string `json:"src_path"` + DestPath string `json:"dest_path"` + ReadOnly bool `json:"read_only"` +} + +type Flavor struct { + Code string `json:"code"` +} + +type CreateTrainJobResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + JobName string `json:"job_name"` + JobID int64 `json:"job_id"` + Status int `json:"status"` + CreateTime int64 `json:"create_time"` + VersionID int64 `json:"version_id"` + ResourceID string `json:"resource_id"` + VersionName string `json:"version_name"` +} + +type CreateTrainJobConfigResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` +} + +type GetResourceSpecsResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + SpecTotalCount int `json:"spec_total_count"` + Specs []Specs `json:"specs"` +} + +type Specs struct { + Core string `json:"core"` + Cpu string `json:"cpu"` + IsNoResource bool `json:"no_resource"` + GpuType string `json:"gpu_type"` + SpecID int64 `json:"spec_id"` + GpuNum int `json:"gpu_num"` + SpecCode string `json:"spec_code"` + Storage string `json:"storage"` + MaxNum int `json:"max_num"` + UnitNum int `json:"unit_num"` + InterfaceType int `json:"interface_type"` +} + +type GetConfigListResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + ConfigTotalCount int `json:"config_total_count"` + ParaConfigs []ParaConfig `json:"configs"` +} + +type ParaConfig struct { + ConfigName string `json:"config_name"` + ConfigDesc string `json:"config_desc"` + CreateTime int64 `json:"create_time"` + EngineType int `json:"engine_type"` + EngineName string `json:"engine_name"` + EngineId int64 `json:"engine_id"` + EngineVersion string `json:"engine_version"` + UserImageUrl string `json:"user_image_url"` + UserCommand string `json:"user_command"` + Result GetConfigResult +} + +type GetConfigResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + ConfigName string `json:"config_name"` + Description string `json:"config_desc"` + WorkServerNum int `json:"worker_server_num"` + AppUrl string `json:"app_url"` //训练作业的代码目录 + BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 + Parameter []Parameter `json:"parameter"` + DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL + //DatasetID string `json:"dataset_id"` + //DataVersionID string `json:"dataset_version_id"` + //DataSource []DataSource `json:"data_source"` + //SpecID int64 `json:"spec_id"` + EngineID int64 `json:"engine_id"` + //ModelID int64 `json:"model_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + //UserImageUrl string `json:"user_image_url"` + //UserCommand string `json:"user_command"` + //CreateVersion bool `json:"create_version"` + //Volumes []Volumes `json:"volumes"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` +} + +type ErrorResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_message"` + IsSuccess bool `json:"is_success"` +} + +type GetTrainJobResult struct { + IsSuccess bool `json:"is_success"` + JobName string `json:"job_name"` + JobID int64 `json:"job_id"` + Description string `json:"job_desc"` + IntStatus int `json:"status"` + Status string + LongCreateTime int64 `json:"create_time"` + CreateTime string + Duration int64 `json:"duration"` //训练作业的运行时间，单位为毫秒 + TrainJobDuration string //训练作业的运行时间，格式为hh:mm:ss + VersionID int64 `json:"version_id"` + ResourceID string `json:"resource_id"` + VersionName string `json:"version_name"` + PreVersionID int64 `json:"pre_version_id"` + WorkServerNum int `json:"worker_server_num"` + AppUrl string `json:"app_url"` //训练作业的代码目录 + BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 + Parameter []Parameter `json:"parameter"` + DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL + //DatasetID string `json:"dataset_id"` + //DataVersionID string `json:"dataset_version_id"` + //DataSource []DataSource `json:"data_source"` + //SpecID int64 `json:"spec_id"` + EngineID int64 `json:"engine_id"` + EngineName string `json:"engine_name"` + EngineVersion string `json:"engine_version"` + //ModelID int64 `json:"model_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + //UserImageUrl string `json:"user_image_url"` + //UserCommand string `json:"user_command"` + //Volumes []Volumes `json:"volumes"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` + PoolName string `json:"pool_name"` + NasMountPath string `json:"nas_mount_path"` + NasShareAddr string `json:"nas_share_addr"` + DatasetName string +} + +type GetTrainJobLogResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + Content string `json:"content"` + Lines int `json:"lines"` + StartLine string `json:"start_line"` + EndLine string `json:"end_line"` +} + +type GetTrainJobLogFileNamesResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` + LogFileList []string `json:"log_file_list"` +} + +type TrainJobResult struct { + ErrorCode string `json:"error_code"` + ErrorMsg string `json:"error_msg"` + IsSuccess bool `json:"is_success"` +} + +type LogFile struct { + Name string +} + func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { sess := x.NewSession() defer sess.Close() @@ -559,6 +848,12 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { ) } + if (opts.JobType) != "" { + cond = cond.And( + builder.Eq{"cloudbrain.job_type": opts.JobType}, + ) + } + // switch opts.JobStatus { // case JobWaiting: // cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) @@ -647,6 +942,12 @@ func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err erro return } +func SetTrainJobStatusByJobID(jobID string, status string, duration int64, trainjobduration string) (err error) { + cb := &Cloudbrain{JobID: jobID, Status: string(status), Duration: duration, TrainJobDuration: trainjobduration} + _, err = x.Cols("status", "duration", "train_job_duration").Where("cloudbrain.job_id=?", jobID).Update(cb) + return +} + func UpdateJob(job *Cloudbrain) error { return updateJob(x, job) } @@ -658,6 +959,17 @@ func updateJob(e Engine, job *Cloudbrain) error { return err } +// func UpdateTrainJob(job *CloudbrainInfo) error { +// return updateTrainJob(x, job) +// } + +// func updateTrainJob(e Engine, job *CloudbrainInfo) error { +// var sess *xorm.Session +// sess = e.Where("job_id = ?", job.Cloudbrain.JobID) +// _, err := sess.Cols("status", "container_id", "container_ip").Update(job) +// return err +// } + func DeleteJob(job *Cloudbrain) error { return deleteJob(x, job) } @@ -673,7 +985,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { - if !isSigned || job.Status != string(JobStopped) { + if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { return false } repo, err := GetRepositoryByID(job.RepoID) diff --git a/models/dataset.go b/models/dataset.go index e7160006d..402a548ef 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -139,7 +139,14 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { if opts.IncludePublic { cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) if opts.OwnerID > 0 { - cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID}) + if len(opts.Keyword) == 0 { + cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID}) + } else { + subCon := builder.NewCond() + subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID}, builder.Like{"dataset.title", opts.Keyword}) + cond = cond.Or(subCon) + + } } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) diff --git a/models/file_chunk.go b/models/file_chunk.go index b849f0108..76c926dc5 100755 --- a/models/file_chunk.go +++ b/models/file_chunk.go @@ -14,8 +14,8 @@ const ( ) const ( - TypeCloudBrainOne = 0 - TypeCloudBrainTwo = 1 + TypeCloudBrainOne int = iota + TypeCloudBrainTwo ) type FileChunk struct { diff --git a/models/models.go b/models/models.go index 412148235..696d0949b 100755 --- a/models/models.go +++ b/models/models.go @@ -137,7 +137,9 @@ func init() { tablesStatistic = append(tablesStatistic, new(RepoStatistic), + new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserLoginLog), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 7f4bfebba..c8629875e 100755 --- a/models/repo.go +++ b/models/repo.go @@ -6,13 +6,14 @@ package models import ( - "code.gitea.io/gitea/modules/blockchain" "context" "crypto/md5" "errors" "fmt" "html/template" + "code.gitea.io/gitea/modules/blockchain" + // Needed for jpeg support _ "image/jpeg" "image/png" @@ -171,11 +172,11 @@ type Repository struct { NumOpenIssues int `xorm:"-"` NumPulls int NumClosedPulls int - NumOpenPulls int `xorm:"-"` - NumMilestones int `xorm:"NOT NULL DEFAULT 0"` - NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` - NumOpenMilestones int `xorm:"-"` - NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` + NumOpenPulls int `xorm:"-"` + NumMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumOpenMilestones int `xorm:"-"` + NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` @@ -215,8 +216,8 @@ type Repository struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Hot int64 `xorm:"-"` - Active int64 `xorm:"-"` + Hot int64 `xorm:"-"` + Active int64 `xorm:"-"` } // SanitizedOriginalURL returns a sanitized OriginalURL @@ -1430,6 +1431,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) { } +func GetAllRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Count(repo) +} + +func GetAllRepositoriesSize() (int64, error) { + return x.SumInt(&Repository{}, "size") +} + func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) @@ -2464,7 +2474,7 @@ func (repo *Repository) IncreaseCloneCnt() { } func UpdateRepositoryCommitNum(repo *Repository) error { - if _,err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil { + if _, err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil { return err } diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index f6cbf0331..9cb7e4a09 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -1,13 +1,115 @@ package models -import "code.gitea.io/gitea/modules/git" +import ( + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/modules/git" +) func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) { wikiPath := "" if repo.HasWiki() { wikiPath = repo.WikiPath() } - return git.GetRepoKPIStats(repo.RepoPath(), wikiPath) + return getRepoKPIStats(repo.RepoPath(), wikiPath) +} + +func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error) { + stats := &git.RepoKPIStats{} + + contributors, err := git.GetContributors(repoPath) + if err != nil { + return nil, err + } + timeUntil := time.Now() + fourMonthAgo := timeUntil.AddDate(0, -4, 0) + recentlyContributors, err := git.GetContributorsDetail(repoPath, fourMonthAgo) + newContributersDict := make(map[string]struct{}) + if err != nil { + return nil, err + } + + if contributors != nil { + contributorDistinctDict := make(map[string]int, 0) + keyContributorsDict := make(map[string]struct{}, 0) + + for _, contributor := range contributors { + if strings.Compare(contributor.Email, "") == 0 { + continue + } + + user, err := GetUserByActivateEmail(contributor.Email) + if err == nil { + value, ok := contributorDistinctDict[user.Email] + if !ok { + contributorDistinctDict[user.Email] = contributor.CommitCnt + } else { + contributorDistinctDict[user.Email] = value + contributor.CommitCnt + } + setKeyContributerDict(contributorDistinctDict, user.Email, keyContributorsDict) + + } else { + value, ok := contributorDistinctDict[contributor.Email] + if !ok { + contributorDistinctDict[contributor.Email] = contributor.CommitCnt + } else { + contributorDistinctDict[contributor.Email] = value + contributor.CommitCnt + } + setKeyContributerDict(contributorDistinctDict, contributor.Email, keyContributorsDict) + } + + } + + if recentlyContributors != nil { + for _, recentlyContributor := range recentlyContributors { + + user, err := GetUserByActivateEmail(recentlyContributor.Email) + var ok bool + if err == nil { + _, ok = contributorDistinctDict[user.Email] + } else { + _, ok = contributorDistinctDict[recentlyContributor.Email] + } + + if !ok { + stats.ContributorsAdded++ + newContributersDict[recentlyContributor.Email] = struct{}{} + } + + } + } + + stats.Contributors = int64(len(contributorDistinctDict)) + stats.KeyContributors = int64(len(keyContributorsDict)) + + } + + err = git.SetDevelopAge(repoPath, stats) + if err != nil { + return nil, fmt.Errorf("FillFromGit: %v", err) + } + err = git.SetRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict) + + if err != nil { + return nil, fmt.Errorf("FillFromGit: %v", err) + } + + git.SetWikiPages(wikiPath, stats) + return stats, nil + +} + +func setKeyContributerDict(contributorDistinctDict map[string]int, email string, keyContributorsDict map[string]struct{}) { + if contributorDistinctDict[email] >= 3 { + _, ok := keyContributorsDict[email] + if !ok { + keyContributorsDict[email] = struct{}{} + + } + + } } func GetAllUserKPIStats() (map[string]*git.UserKPIStats, error) { diff --git a/models/repo_statistic.go b/models/repo_statistic.go index b987f4f46..adef672e0 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -1,38 +1,64 @@ package models import ( - "code.gitea.io/gitea/modules/timeutil" "fmt" + "time" + + "code.gitea.io/gitea/modules/timeutil" ) // RepoStatistic statistic info of all repository type RepoStatistic struct { - ID int64 `xorm:"pk autoincr"` - RepoID int64 `xorm:"unique(s) NOT NULL"` - Date string `xorm:"unique(s) NOT NULL"` - NumWatches int64 `xorm:"NOT NULL DEFAULT 0"` - NumStars int64 `xorm:"NOT NULL DEFAULT 0"` - NumForks int64 `xorm:"NOT NULL DEFAULT 0"` - NumDownloads int64 `xorm:"NOT NULL DEFAULT 0"` - NumComments int64 `xorm:"NOT NULL DEFAULT 0"` - NumVisits int64 `xorm:"NOT NULL DEFAULT 0"` - NumClosedIssues int64 `xorm:"NOT NULL DEFAULT 0"` - NumVersions int64 `xorm:"NOT NULL DEFAULT 0"` - //develop months - NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0"` - RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` - DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` - NumModels int64 `xorm:"NOT NULL DEFAULT 0"` - NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0"` - NumCommits int64 `xorm:"NOT NULL DEFAULT 0"` - NumIssues int64 `xorm:"NOT NULL DEFAULT 0"` - NumPulls int64 `xorm:"NOT NULL DEFAULT 0"` - IssueFixedRate float32 `xorm:"NOT NULL"` - NumContributor int64 `xorm:"NOT NULL DEFAULT 0"` - NumKeyContributor int64 `xorm:"NOT NULL DEFAULT 0"` - - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"unique(s) NOT NULL"` + Name string `xorm:"INDEX"` + IsPrivate bool + Date string `xorm:"unique(s) NOT NULL"` + NumWatches int64 `xorm:"NOT NULL DEFAULT 0"` + NumWatchesAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumStars int64 `xorm:"NOT NULL DEFAULT 0"` + NumStarsAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumForks int64 `xorm:"NOT NULL DEFAULT 0"` + NumForksAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumDownloads int64 `xorm:"NOT NULL DEFAULT 0"` + NumDownloadsAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumComments int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommentsAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumVisits int64 `xorm:"NOT NULL DEFAULT 0"` + NumClosedIssues int64 `xorm:"NOT NULL DEFAULT 0"` + NumClosedIssuesAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumVersions int64 `xorm:"NOT NULL DEFAULT 0"` + NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommits int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommitsAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumIssues int64 `xorm:"NOT NULL DEFAULT 0"` + NumIssuesAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumPulls int64 `xorm:"NOT NULL DEFAULT 0"` + NumPullsAdded int64 `xorm:"NOT NULL DEFAULT 0"` + IssueFixedRate float32 `xorm:"NOT NULL"` + NumContributor int64 `xorm:"NOT NULL DEFAULT 0"` + NumContributorAdded int64 `xorm:"NOT NULL DEFAULT 0"` + NumKeyContributor int64 `xorm:"NOT NULL DEFAULT 0"` + + NumContributorsGrowth int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommitsGrowth int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommitLinesGrowth int64 `xorm:"NOT NULL DEFAULT 0"` + NumIssuesGrowth int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommentsGrowth int64 `xorm:"NOT NULL DEFAULT 0"` + + Impact float64 `xorm:"NOT NULL DEFAULT 0"` + Completeness float64 `xorm:"NOT NULL DEFAULT 0"` + Liveness float64 `xorm:"NOT NULL DEFAULT 0"` + ProjectHealth float64 `xorm:"NOT NULL DEFAULT 0"` + TeamHealth float64 `xorm:"NOT NULL DEFAULT 0"` + Growth float64 `xorm:"NOT NULL DEFAULT 0"` + RadarTotal float64 `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } func DeleteRepoStatDaily(date string) error { @@ -55,6 +81,35 @@ func DeleteRepoStatDaily(date string) error { return nil } +func GetRepoStatisticByDate(date string) ([]*RepoStatistic, error) { + repoStatistics := make([]*RepoStatistic, 0) + err := xStatistic.Where("date = ?", date).Find(&repoStatistics) + return repoStatistics, err + +} + +func GetOneRepoStatisticBeforeTime(time time.Time) (*RepoStatistic, error) { + repoStatistics := make([]*RepoStatistic, 0) + err := xStatistic.Where("created_unix >= ?", time.Unix()).OrderBy("created_unix").Limit(1).Find(&repoStatistics) + if err != nil { + return nil, err + } else { + if len(repoStatistics) == 0 { + return nil, fmt.Errorf("the repo statistic record count is 0") + } else { + return repoStatistics[0], nil + } + } + +} + func InsertRepoStat(repoStat *RepoStatistic) (int64, error) { return xStatistic.Insert(repoStat) } + +func UpdateRepoStat(repoStat *RepoStatistic) error { + sql := "update repo_statistic set impact=?,completeness=?,liveness=?,project_health=?,team_health=?,growth=?,radar_total=? where repo_id=? and date=?" + + _, err := xStatistic.Exec(sql, repoStat.Impact, repoStat.Completeness, repoStat.Liveness, repoStat.ProjectHealth, repoStat.TeamHealth, repoStat.Growth, repoStat.RadarTotal, repoStat.RepoID, repoStat.Date) + return err +} diff --git a/models/summary_statistic.go b/models/summary_statistic.go new file mode 100644 index 000000000..0addd472b --- /dev/null +++ b/models/summary_statistic.go @@ -0,0 +1,69 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" +) + +var DomainMap = map[string]int{ + "大模型": 0, + "ai开发工具": 1, + "计算机视觉": 2, + "自然语言处理": 3, + "机器学习": 4, + "神经网络": 5, + "自动驾驶": 6, + "机器人": 7, + "联邦学习": 8, + "数据挖掘": 9, + "risc-v开发": 10, +} + +type SummaryStatistic struct { + ID int64 `xorm:"pk autoincr"` + Date string `xorm:"unique(s) NOT NULL"` + NumUsers int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumOrganizations int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepos int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoBigModel int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAI int `xorm:"NOT NULL DEFAULT 0"` + NumRepoVision int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNLP int `xorm:"NOT NULL DEFAULT 0"` + NumRepoML int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNN int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAutoDrive int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRobot int `xorm:"NOT NULL DEFAULT 0"` + NumRepoLeagueLearn int `xorm:"NOT NULL DEFAULT 0"` + NumRepoDataMining int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRISC int `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func DeleteSummaryStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { + return xStatistic.Insert(summaryStatistic) +} diff --git a/models/topic.go b/models/topic.go index b8d3d9d85..5533da7bc 100644 --- a/models/topic.go +++ b/models/topic.go @@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) { return &topic, nil } +func GetAllUsedTopics() ([]*Topic, error) { + topics := make([]*Topic, 0) + err := x.Where("repo_count > ?", 0).Find(&topics) + return topics, err + +} + // addTopicByNameToRepo adds a topic name to a repo and increments the topic count. // Returns topic after the addition func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) { @@ -178,7 +185,7 @@ func (opts *FindTopicOptions) toConds() builder.Cond { } if opts.Keyword != "" { - cond = cond.And(builder.Like{"topic.name", opts.Keyword}) + cond = cond.And(builder.Like{"topic.name", strings.ToLower(opts.Keyword)}) } return cond diff --git a/models/user.go b/models/user.go index 78ab4627a..1ee20d74c 100755 --- a/models/user.go +++ b/models/user.go @@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { return nil } +func GetUsersCount() (int64, error) { + user := new(User) + return x.Where("type=0").Count(user) + +} + +func GetOrganizationsCount() (int64, error) { + user := new(User) + return x.Where("type=1").Count(user) + +} + func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) err := x.Where("public_key = ''"). diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a7d549cd4..bb6726a2c 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -71,7 +71,50 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` } -func CountData(wikiCountMap map[string]int) { +func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { + log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") + + userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) + statictisSess.Find(&userBusinessAnalysisList) + + resultMap := make(map[int64]*UserBusinessAnalysis) + log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) + for _, userRecord := range userBusinessAnalysisList { + if _, ok := resultMap[userRecord.ID]; !ok { + resultMap[userRecord.ID] = userRecord + } else { + resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount + resultMap[userRecord.ID].CommitCount += userRecord.CommitCount + resultMap[userRecord.ID].IssueCount += userRecord.IssueCount + resultMap[userRecord.ID].CommentCount += userRecord.CommentCount + resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount + resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount + resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount + resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize + resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount + resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount + resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount + resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount + resultMap[userRecord.ID].LoginCount += userRecord.LoginCount + } + } + + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap)) + index := 0 + for _, v := range resultMap { + userBusinessAnalysisReturnList[index] = v + index += 1 + } + log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) + return userBusinessAnalysisReturnList +} + +func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() @@ -82,17 +125,17 @@ func CountData(wikiCountMap map[string]int) { currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - yesterday := currentTimeNow.AddDate(0, 0, -1) - startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) + //yesterday := currentTimeNow.AddDate(0, 0, -1) + //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) + //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) - CodeMergeCountMap := queryAction(start_unix, end_unix, 11) + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryAction(start_unix, end_unix, 5) IssueCountMap := queryAction(start_unix, end_unix, 10) @@ -110,12 +153,19 @@ func CountData(wikiCountMap map[string]int) { CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() for i, userRecord := range userList { var dateRecord UserBusinessAnalysis dateRecord.ID = userRecord.ID log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + dateRecord.Email = userRecord.Email dateRecord.RegistDate = userRecord.CreatedUnix dateRecord.Name = userRecord.Name @@ -192,10 +242,14 @@ func CountData(wikiCountMap map[string]int) { dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } + dateRecord.CommitModelCount = 0 - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() statictisSess.Insert(&dateRecord) } @@ -223,6 +277,28 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { } +func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + + sess.Select("issue.*").Table("issue"). + Join("inner", "pull_request", "issue.id=pull_request.issue_id"). + Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) + + issueList := make([]*Issue, 0) + sess.Find(&issueList) + resultMap := make(map[int64]int) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + return resultMap +} + func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() @@ -341,7 +417,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) repoList := make([]*Repository, 0) sess.Find(&repoList) resultMap := make(map[int64]int) @@ -354,7 +430,24 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { } } return resultMap +} +func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + return resultMap } func subMonth(t1, t2 time.Time) (month int) { diff --git a/models/user_login_log.go b/models/user_login_log.go new file mode 100644 index 000000000..4a499d527 --- /dev/null +++ b/models/user_login_log.go @@ -0,0 +1,34 @@ +package models + +import ( + "net/http" + + "code.gitea.io/gitea/modules/timeutil" +) + +type UserLoginLog struct { + ID int64 `xorm:"pk autoincr"` + UId int64 `xorm:"NOT NULL"` + IpAddr string `xorm:"default NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` +} + +func SaveLoginInfoToDb(r *http.Request, u *User) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + + var dateRecord UserLoginLog + + dateRecord.UId = u.ID + dateRecord.IpAddr = getIP(r) + + statictisSess.Insert(&dateRecord) +} + +func getIP(r *http.Request) string { + forwarded := r.Header.Get("X-FORWARDED-FOR") + if forwarded != "" { + return forwarded + } + return r.RemoteAddr +} diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 0be3e3882..f2e5aeed5 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -14,3 +14,32 @@ type CreateModelArtsForm struct { func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { return validate(errs, ctx.Data, f, ctx.Locale) } + +type CreateModelArtsNotebookForm struct { + JobName string `form:"job_name" binding:"Required"` + Attachment string `form:"attachment"` + Description string `form:"description"` +} + +func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} + +type CreateModelArtsTrainJobForm struct { + JobName string `form:"job_name" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + BootFile string `form:"boot_file" binding:"Required"` + WorkServerNumber int `form:"work_server_number" binding:"Required"` + EngineID int `form:"engine_id" binding:"Required"` + PoolID string `form:"pool_id" binding:"Required"` + Flavor string `form:"flavor" binding:"Required"` + Params string `form:"run_para_list" binding:"Required"` + Description string `form:"description"` + IsSaveParam string `form:"is_save_para"` + ParameterTemplateName string `form:"parameter_template_name"` + PrameterDescription string `form:"parameter_description"` +} + +func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/modules/base/tool.go b/modules/base/tool.go index 8145522e2..cf2972990 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -224,7 +224,7 @@ func SizedAvatarLinkWithDomain(email string, size int) string { // FileSize calculates the file size and generate user-friendly string. func FileSize(s int64) string { - return humanize.IBytes(uint64(s)) + return humanize.Bytes(uint64(s)) } // PrettyNumber produces a string form of the given number in base 10 with diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 0de1db9a6..8f6bf4e17 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -16,6 +16,7 @@ const ( ModelMountPath = "/model" BenchMarkMountPath = "/benchmark" Snn4imagenetMountPath = "/snn4imagenet" + BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" SubTaskName = "task1" @@ -27,7 +28,7 @@ var ( ResourceSpecs *models.ResourceSpecs ) -func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue string, resourceSpecId int) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + @@ -103,6 +104,13 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, ReadOnly: true, }, }, + { + HostPath: models.StHostPath{ + Path: brainScorePath, + MountPath: BrainScoreMountPath, + ReadOnly: true, + }, + }, }, }) if err != nil { @@ -123,7 +131,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, - Type: models.TypeCloudBrainOne, + Type: models.TypeCloudBrainOne, + Uuid: uuid, }) if err != nil { diff --git a/modules/context/context.go b/modules/context/context.go old mode 100644 new mode 100755 index 71c8986fb..6877780e3 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -310,9 +310,11 @@ func Contexter() macaron.Handler { ctx.Data["SignedUserID"] = ctx.User.ID ctx.Data["SignedUserName"] = ctx.User.Name ctx.Data["IsAdmin"] = ctx.User.IsAdmin + c.Data["SignedUserName"] = ctx.User.Name } else { ctx.Data["SignedUserID"] = int64(0) ctx.Data["SignedUserName"] = "" + c.Data["SignedUserName"] = "" } // If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid. diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 26cd16778..ed9829cef 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -174,6 +174,16 @@ func registerHandleRepoStatistic() { }) } +func registerHandleSummaryStatistic() { + RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SummaryStatistic() + return nil + }) +} func registerHandleUserStatistic() { RegisterTaskFatal("handle_user_statistic", &BaseConfig{ Enabled: true, @@ -202,4 +212,5 @@ func initBasicTasks() { registerHandleRepoStatistic() registerHandleUserStatistic() + registerHandleSummaryStatistic() } diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go index f7556d5c2..5d99bd8af 100644 --- a/modules/git/repo_stats_custom.go +++ b/modules/git/repo_stats_custom.go @@ -35,58 +35,7 @@ type UserKPITypeStats struct { isNewContributor bool //是否是4个月内的新增贡献者 } -func GetRepoKPIStats(repoPath string, wikiPath string) (*RepoKPIStats, error) { - stats := &RepoKPIStats{} - - contributors, err := GetContributors(repoPath) - if err != nil { - return nil, err - } - timeUntil := time.Now() - fourMonthAgo := timeUntil.AddDate(0, -4, 0) - recentlyContributors, err := getContributors(repoPath, fourMonthAgo) - newContributersDict := make(map[string]struct{}) - if err != nil { - return nil, err - } - - if contributors != nil { - stats.Contributors = int64(len(contributors)) - for _, contributor := range contributors { - if contributor.CommitCnt >= 3 { - stats.KeyContributors++ - } - - if recentlyContributors != nil { - for _, recentlyContributor := range recentlyContributors { - if recentlyContributor.Email == contributor.Email && recentlyContributor.CommitCnt == contributor.CommitCnt { - stats.ContributorsAdded++ - newContributersDict[recentlyContributor.Email] = struct{}{} - } - - } - } - - } - - } - - err = setDevelopAge(repoPath, stats) - if err != nil { - return nil, fmt.Errorf("FillFromGit: %v", err) - } - err = setRepoKPIStats(repoPath, fourMonthAgo, stats, newContributersDict) - - if err != nil { - return nil, fmt.Errorf("FillFromGit: %v", err) - } - - setWikiPages(wikiPath, stats) - return stats, nil - -} - -func setDevelopAge(repoPath string, stats *RepoKPIStats) error { +func SetDevelopAge(repoPath string, stats *RepoKPIStats) error { args := []string{"log", "--no-merges", "--branches=*", "--format=%cd", "--date=short"} stdout, err := NewCommand(args...).RunInDirBytes(repoPath) if err != nil { @@ -173,7 +122,7 @@ func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { } -func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { +func SetRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { since := fromTime.Format(time.RFC3339) args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} @@ -259,7 +208,7 @@ func setRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, n } -func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) { +func GetContributorsDetail(repoPath string, fromTime time.Time) ([]Contributor, error) { since := fromTime.Format(time.RFC3339) cmd := NewCommand("shortlog", "-sne", "--all", fmt.Sprintf("--since='%s'", since)) stdout, err := cmd.RunInDir(repoPath) @@ -289,7 +238,7 @@ func getContributors(repoPath string, fromTime time.Time) ([]Contributor, error) return nil, nil } -func setWikiPages(wikiPath string, stats *RepoKPIStats) { +func SetWikiPages(wikiPath string, stats *RepoKPIStats) { wikiPages := 0 if wikiPath == "" { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index edd9d5d6b..e1dbe9f5a 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,22 +1,53 @@ package modelarts import ( + "encoding/json" + "path" + "strconv" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - "encoding/json" - "path" ) const ( + //notebook storageTypeOBS = "obs" autoStopDuration = 4 * 60 * 60 DataSetMountPath = "/home/ma-user/work" NotebookEnv = "Python3" NotebookType = "Ascend" + FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB （modelarts.kat1.xlarge）" + + //train-job + // ResourcePools = "{\"resource_pool\":[{\"id\":\"pool1328035d\", \"value\":\"专属资源池\"}]}" + // Engines = "{\"engine\":[{\"id\":1, \"value\":\"Ascend-Powered-Engine\"}]}" + // EngineVersions = "{\"version\":[{\"id\":118,\"value\":\"MindSpore-1.0.0-c75-python3.7-euleros2.8-aarch64\"}," + + // "{\"id\":119,\"value\":\"MindSpore-1.1.1-c76-python3.7-euleros2.8-aarch64\"}," + + // "{\"id\":120,\"value\":\"MindSpore-1.1.1-c76-tr5-python3.7-euleros2.8-aarch64\"}," + + // "{\"id\":117,\"value\":\"TF-1.15-c75-python3.7-euleros2.8-aarch64\"}" + + // "]}" + // TrainJobFlavorInfo = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU：48 核 512GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU：192 核 2048GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU：96 核 1024GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU：24 核 256GiB\"}" + + // "]}" + CodePath = "/code/" + OutputPath = "/output/" + LogPath = "/log/" + JobPath = "/job/" + OrderDesc = "desc" //向下查询 + OrderAsc = "asc" //向上查询 + Lines = 20 + TrainUrl = "train_url" + DataUrl = "data_url" + PerPage = 10 + + SortByCreateTime = "create_time" + ConfigTypeCustom = "custom" ) var ( @@ -24,6 +55,50 @@ var ( FlavorInfos *models.FlavorInfos ) +type GenerateTrainJobReq struct { + JobName string + Uuid string + Description string + CodeObsPath string + BootFile string + DataUrl string + TrainUrl string + FlavorCode string + LogUrl string + PoolID string + WorkServerNumber int + EngineID int64 + Parameters []models.Parameter +} + +type VersionInfo struct { + Version []struct { + ID int `json:"id"` + Value string `json:"value"` + } `json:"version"` +} + +type Flavor struct { + Info []struct { + Code string `json:"code"` + Value string `json:"value"` + } `json:"flavor"` +} + +type Engine struct { + Info []struct { + ID int `json:"id"` + Value string `json:"value"` + } `json:"engine"` +} + +type ResourcePool struct { + Info []struct { + ID string `json:"id"` + Value string `json:"value"` + } `json:"resource_pool"` +} + func GenerateTask(ctx *context.Context, jobName, uuid, description string) error { var dataActualPath string if uuid != "" { @@ -76,8 +151,8 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error log.Error("CreateJob failed: %v", err.Error()) return err } - err = models.CreateCloudbrain(&models.Cloudbrain{ + Status: string(models.JobWaiting), UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, @@ -85,6 +160,7 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error JobName: jobName, JobType: string(models.JobTypeDebug), Type: models.TypeCloudBrainTwo, + Uuid: uuid, }) if err != nil { @@ -93,3 +169,110 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error return nil } + +func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) error { + jobResult, err := createTrainJob(models.CreateTrainJobParams{ + JobName: req.JobName, + Description: req.Description, + Config: models.Config{ + WorkServerNum: req.WorkServerNumber, + AppUrl: req.CodeObsPath, + BootFileUrl: req.BootFile, + DataUrl: req.DataUrl, + EngineID: req.EngineID, + TrainUrl: req.TrainUrl, + LogUrl: req.LogUrl, + PoolID: req.PoolID, + CreateVersion: true, + Flavor: models.Flavor{ + Code: req.FlavorCode, + }, + Parameter: req.Parameters, + }, + }) + if err != nil { + log.Error("CreateJob failed: %v", err.Error()) + return err + } + + attach, err := models.GetAttachmentByUUID(req.Uuid) + if err != nil { + log.Error("GetAttachmentByUUID(%s) failed:%v", strconv.FormatInt(jobResult.JobID, 10), err.Error()) + return nil + } + + err = models.CreateCloudbrain(&models.Cloudbrain{ + Status: TransTrainJobStatus(jobResult.Status), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: strconv.FormatInt(jobResult.JobID, 10), + JobName: req.JobName, + JobType: string(models.JobTypeTrain), + Type: models.TypeCloudBrainTwo, + VersionID: jobResult.VersionID, + VersionName: jobResult.VersionName, + Uuid: req.Uuid, + DatasetName: attach.Name, + }) + + if err != nil { + log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) + return err + } + + return nil +} + +func TransTrainJobStatus(status int) string { + switch status { + case 0: + return "UNKNOWN" + case 1: + return "INIT" + case 2: + return "IMAGE_CREATING" + case 3: + return "IMAGE_FAILED" + case 4: + return "SUBMIT_TRYING" + case 5: + return "SUBMIT_FAILED" + case 6: + return "DELETE_FAILED" + case 7: + return "WAITING" + case 8: + return "RUNNING" + case 9: + return "KILLING" + case 10: + return "COMPLETED" + case 11: + return "FAILED" + case 12: + return "KILLED" + case 13: + return "CANCELED" + case 14: + return "LOST" + case 15: + return "SCALING" + case 16: + return "SUBMIT_MODEL_FAILED" + case 17: + return "DEPLOY_SERVICE_FAILED" + case 18: + return "CHECK_INIT" + case 19: + return "CHECK_RUNNING" + case 20: + return "CHECK_RUNNING_COMPLETED" + case 21: + return "CHECK_FAILED" + + default: + return strconv.Itoa(status) + } + + return "" +} diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index f91be5e31..d17478c94 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -1,13 +1,14 @@ package modelarts import ( - "code.gitea.io/gitea/modules/log" "crypto/tls" "encoding/json" "fmt" "net/http" + "strconv" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "github.com/go-resty/resty/v2" ) @@ -23,6 +24,9 @@ const ( urlGetToken = "/v3/auth/tokens" urlNotebook = "/demanager/instances" + urlTrainJob = "/training-jobs" + urlResourceSpecs = "/job/resource-specs" + urlTrainJobConfig = "/training-job-configs" errorCodeExceedLimit = "ModelArts.0118" ) @@ -104,7 +108,7 @@ sendjob: Post(HOST + "/v1/" + setting.ProjectID + urlNotebook) if err != nil { - return nil, fmt.Errorf("resty create job: %s", err) + return nil, fmt.Errorf("resty create notebook: %s", err) } if res.StatusCode() == http.StatusUnauthorized && retry < 1 { @@ -121,11 +125,11 @@ sendjob: } if len(response.ErrorCode) != 0 { - log.Error("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + log.Error("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg) if response.ErrorCode == errorCodeExceedLimit { response.ErrorMsg = "所选规格使用数量已超过最大配额限制。" } - return &result, fmt.Errorf("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + return &result, fmt.Errorf("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg) } return &result, nil @@ -210,6 +214,45 @@ sendjob: return &result, nil } +func DelNotebook(jobID string) (*models.NotebookDelResult, error) { + checkSetting() + client := getRestyClient() + var result models.NotebookDelResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/v1/" + setting.ProjectID + urlNotebook + "/" + jobID) + + if err != nil { + return &result, fmt.Errorf("resty DelJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + var response models.NotebookResult + err = json.Unmarshal(res.Body(), &response) + if err != nil { + log.Error("json.Unmarshal failed: %s", err.Error()) + return &result, fmt.Errorf("son.Unmarshal failed: %s", err.Error()) + } + + if len(response.ErrorCode) != 0 { + log.Error("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + return &result, fmt.Errorf("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + } + + return &result, nil +} + func DelJob(jobID string) (*models.NotebookDelResult, error) { checkSetting() client := getRestyClient() @@ -287,3 +330,441 @@ sendjob: return &result, nil } + +func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.CreateTrainJobResult, error) { + checkSetting() + client := getRestyClient() + var result models.CreateTrainJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(createJobParams). + SetResult(&result). + Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob) + + if err != nil { + return nil, fmt.Errorf("resty create train-job: %s", err) + } + + req, _ := json.Marshal(createJobParams) + log.Info("%s", req) + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return &result, fmt.Errorf("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func GetResourceSpecs() (*models.GetResourceSpecsResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetResourceSpecsResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlResourceSpecs) + + if err != nil { + return nil, fmt.Errorf("resty GetResourceSpecs: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return &result, fmt.Errorf("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func CreateTrainJobConfig(req models.CreateConfigParams) (*models.CreateTrainJobConfigResult, error) { + checkSetting() + client := getRestyClient() + var result models.CreateTrainJobConfigResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(req). + SetResult(&result). + Post(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig) + + if err != nil { + return nil, fmt.Errorf("resty CreateTrainJobConfig: %s", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + //temp, _ := json.Marshal(req) + //log.Info("%s", temp) + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return &result, fmt.Errorf("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func GetConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetConfigListResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetQueryParams(map[string]string{ + "per_page": strconv.Itoa(perPage), + "page": strconv.Itoa(page), + "sortBy": sortBy, + "order": order, + "search_content": searchContent, + "config_type": configType, + }). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig) + + if err != nil { + return nil, fmt.Errorf("resty GetConfigList: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetConfigList failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("获取参数配置列表失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetConfigList failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return &result, fmt.Errorf("获取参数配置列表失败(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return &result, nil +} + +func GetParaConfig(configName, configType string) (models.GetConfigResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetConfigResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetQueryParams(map[string]string{ + "config_type": configType, + }). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig + "/" + configName) + + if err != nil { + return result, fmt.Errorf("resty GetParaConfig: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetParaConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return result, fmt.Errorf("获取参数配置详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetParaConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg) + return result, fmt.Errorf("获取参数配置详情失败(%s): %s", result.ErrorCode, result.ErrorMsg) + } + + return result, nil +} + +func GetTrainJob(jobID, versionID string) (*models.GetTrainJobResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetTrainJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID) + + if err != nil { + return nil, fmt.Errorf("resty GetTrainJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("获取作业详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetTrainJob(%s) failed", jobID) + return &result, fmt.Errorf("获取作业详情失败") + } + + return &result, nil +} + +func GetTrainJobLog(jobID, versionID, baseLine, logFile, order string, lines int) (*models.GetTrainJobLogResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetTrainJobLogResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetQueryParams(map[string]string{ + "base_line": baseLine, + "lines": strconv.Itoa(lines), + "log_file": logFile, + "order": order, + }). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/aom-log") + + if err != nil { + return nil, fmt.Errorf("resty GetTrainJobLog: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetTrainJobLog failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("获取作业日志失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetTrainJobLog(%s) failed", jobID) + return &result, fmt.Errorf("获取作业日志失败:%s", result.ErrorMsg) + } + + return &result, nil +} + +func GetTrainJobLogFileNames(jobID, versionID string) (*models.GetTrainJobLogFileNamesResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetTrainJobLogFileNamesResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/log/file-names") + + if err != nil { + return nil, fmt.Errorf("resty GetTrainJobLogFileNames: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("GetTrainJobLogFileNames(%s) failed", jobID) + return &result, fmt.Errorf("获取作业日志文件失败:%s", result.ErrorMsg) + } + + return &result, nil +} + +func DelTrainJob(jobID string) (*models.TrainJobResult, error) { + checkSetting() + client := getRestyClient() + var result models.TrainJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID) + + if err != nil { + return &result, fmt.Errorf("resty DelTrainJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("删除训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("DelTrainJob(%s) failed", jobID) + return &result, fmt.Errorf("删除训练作业失败:%s", result.ErrorMsg) + } + + return &result, nil +} + +func StopTrainJob(jobID, versionID string) (*models.TrainJobResult, error) { + checkSetting() + client := getRestyClient() + var result models.TrainJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/stop") + + if err != nil { + return &result, fmt.Errorf("resty StopTrainJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("StopTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("停止训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("StopTrainJob(%s) failed", jobID) + return &result, fmt.Errorf("停止训练作业失败:%s", result.ErrorMsg) + } + + return &result, nil +} diff --git a/modules/normalization/normalization.go b/modules/normalization/normalization.go new file mode 100644 index 000000000..ce616d7f8 --- /dev/null +++ b/modules/normalization/normalization.go @@ -0,0 +1,83 @@ +package normalization + +import ( + "code.gitea.io/gitea/modules/setting" +) + +func Normalization(value float64, minValue float64, maxValue float64) float64 { + + min := int64(minValue * 100) + max := int64(maxValue * 100) + + if min == max { + return 100.0 + } else { + return 100 * (value - minValue) / (maxValue - minValue) + } + +} + +func GetRadarValue(impactValue float64, completeValue float64, livenessValue float64, projectHealthValue float64, teamHealthValue float64, growthValue float64) float64 { + return setting.RadarMap.Impact*impactValue + + setting.RadarMap.Completeness*completeValue + + setting.RadarMap.Liveness*livenessValue + + setting.RadarMap.ProjectHealth*projectHealthValue + + setting.RadarMap.TeamHealth*teamHealthValue + + setting.RadarMap.Growth*growthValue + +} + +func GetImpactInitValue(watch int64, star int64, fork int64, download int64, comments int64, browser int64) float64 { + + return setting.RadarMap.ImpactWatch*float64(watch) + + setting.RadarMap.ImpactStar*float64(star) + + setting.RadarMap.ImpactFork*float64(fork) + + setting.RadarMap.ImpactCodeDownload*float64(download)*0.001 + + setting.RadarMap.ImpactComments*float64(comments) + + setting.RadarMap.ImpactBrowser*float64(browser)*0.001 + +} + +func GetCompleteInitValue(issuesClosed int64, releases int64, developAge int64, dataset int64, model int64, wiki int64) float64 { + + return setting.RadarMap.CompletenessIssuesClosed*float64(issuesClosed) + + setting.RadarMap.CompletenessReleases*float64(releases) + + setting.RadarMap.CompletenessDevelopAge*float64(developAge) + + setting.RadarMap.CompletenessDataset*(float64(dataset)/(1024*1024)) + + setting.RadarMap.CompletenessModel*float64(model) + + setting.RadarMap.CompletenessWiki*float64(wiki) + +} + +func GetLivenessInitValue(commits int64, issues int64, pr int64, release int64) float64 { + + return setting.RadarMap.LivenessCommit*float64(commits) + + setting.RadarMap.LivenessIssue*float64(issues) + + setting.RadarMap.LivenessPR*float64(pr) + + setting.RadarMap.LivenessRelease*float64(release) + +} + +func GetProjectHealthInitValue(issueClosedRatio float32) float64 { + + return setting.RadarMap.ProjectHealthIssueCompleteRatio * float64(issueClosedRatio) + +} + +func GetTeamHealthInitValue(contributors int64, keyContributors int64, newContributors int64) float64 { + + return setting.RadarMap.TeamHealthContributors*float64(contributors) + + setting.RadarMap.TeamHealthKeyContributors*float64(keyContributors) + + setting.RadarMap.TeamHealthContributorsAdded*float64(newContributors) + +} + +func GetRepoGrowthInitValue(codelinesGrowth int64, issueGrowth int64, commitsGrowth int64, newContributors int64, commentsGrowth int64) float64 { + + return setting.RadarMap.GrowthCodeLines*float64(codelinesGrowth) + + setting.RadarMap.GrowthIssue*float64(issueGrowth) + + setting.RadarMap.GrowthCommit*float64(commitsGrowth) + + setting.RadarMap.GrowthContributors*float64(newContributors) + + setting.RadarMap.GrowthComments*float64(commentsGrowth) + +} diff --git a/modules/setting/radarmap.go b/modules/setting/radarmap.go new file mode 100644 index 000000000..26624d143 --- /dev/null +++ b/modules/setting/radarmap.go @@ -0,0 +1,7 @@ +package setting + +func UpdateRadarMap() { + Cfg.DeleteSection("radar_map") + Cfg.Reload() + SetRadarMapConfig() +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e1e7b7902..eb0e41c90 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -457,18 +457,26 @@ var ( Snn4imagenetCode string Snn4imagenetServerHost string + //snn4imagenet config + IsBrainScoreEnabled bool + BrainScoreCode string + BrainScoreServerHost string + //blockchain config BlockChainHost string CommitValidDate string //obs config - Endpoint string - AccessKeyID string - SecretAccessKey string - Bucket string - Location string - BasePath string - UserBasePath string + Endpoint string + AccessKeyID string + SecretAccessKey string + Bucket string + Location string + BasePath string + OutPutPath string + TrainJobModelPath string + CodePathPrefix string + UserBasePath string //modelarts config ModelArtsHost string @@ -478,10 +486,16 @@ var ( ModelArtsUsername string ModelArtsPassword string ModelArtsDomain string + AllowedOrg string ProfileID string PoolInfos string Flavor string - FlavorInfos string + //train-job + ResourcePools string + Engines string + EngineVersions string + FlavorInfos string + TrainJobFLAVORINFOS string //elk config ElkUrl string @@ -490,6 +504,47 @@ var ( Index string TimeField string ElkTimeFormat string + + //nginx proxy + PROXYURL string + RadarMap = struct { + Impact float64 + ImpactWatch float64 + ImpactStar float64 + ImpactFork float64 + ImpactCodeDownload float64 + ImpactComments float64 + ImpactBrowser float64 + + Completeness float64 + CompletenessIssuesClosed float64 + CompletenessReleases float64 + CompletenessDevelopAge float64 + CompletenessDataset float64 + CompletenessModel float64 + CompletenessWiki float64 + + Liveness float64 + LivenessCommit float64 + LivenessIssue float64 + LivenessPR float64 + LivenessRelease float64 + + ProjectHealth float64 + ProjectHealthIssueCompleteRatio float64 + + TeamHealth float64 + TeamHealthContributors float64 + TeamHealthKeyContributors float64 + TeamHealthContributorsAdded float64 + + Growth float64 + GrowthCodeLines float64 + GrowthIssue float64 + GrowthContributors float64 + GrowthCommit float64 + GrowthComments float64 + }{} ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1181,8 +1236,13 @@ func NewContext() { sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") - Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git") + Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + + sec = Cfg.Section("brainscore") + IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) + BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git") + BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") @@ -1195,7 +1255,11 @@ func NewContext() { Bucket = sec.Key("BUCKET").MustString("testopendata") Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") + TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/") + OutPutPath = sec.Key("Output_Path").MustString("output/") + CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/") + PROXYURL = sec.Key("PROXY_URL").MustString("") sec = Cfg.Section("modelarts") ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") @@ -1205,10 +1269,15 @@ func NewContext() { ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") + AllowedOrg = sec.Key("ORGANIZATION").MustString("") ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") - Flavor = sec.Key("FLAVOR").MustString("") + Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2") + ResourcePools = sec.Key("Resource_Pools").MustString("") + Engines = sec.Key("Engines").MustString("") + EngineVersions = sec.Key("Engine_Versions").MustString("") FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") + TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") sec = Cfg.Section("elk") ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") @@ -1217,6 +1286,45 @@ func NewContext() { Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") + + SetRadarMapConfig() +} + +func SetRadarMapConfig() { + sec := Cfg.Section("radar_map") + + RadarMap.Impact = sec.Key("impact").MustFloat64(0.3) + RadarMap.ImpactWatch = sec.Key("impact_watch").MustFloat64(0.1) + RadarMap.ImpactStar = sec.Key("impact_star").MustFloat64(0.2) + RadarMap.ImpactFork = sec.Key("impact_fork").MustFloat64(0.3) + RadarMap.ImpactCodeDownload = sec.Key("impact_code_download").MustFloat64(0.2) + RadarMap.ImpactComments = sec.Key("impact_comments").MustFloat64(0.1) + RadarMap.ImpactBrowser = sec.Key("impact_browser").MustFloat64(0.1) + RadarMap.Completeness = sec.Key("completeness").MustFloat64(0.1) + RadarMap.CompletenessIssuesClosed = sec.Key("completeness_issues_closed").MustFloat64(0.2) + RadarMap.CompletenessReleases = sec.Key("completeness_releases").MustFloat64(0.3) + RadarMap.CompletenessDevelopAge = sec.Key("completeness_develop_age").MustFloat64(0.1) + RadarMap.CompletenessDataset = sec.Key("completeness_dataset").MustFloat64(0.1) + RadarMap.CompletenessModel = sec.Key("completeness_model").MustFloat64(0.1) + RadarMap.CompletenessWiki = sec.Key("completeness_wiki").MustFloat64(0.1) + RadarMap.Liveness = sec.Key("liveness").MustFloat64(0.3) + RadarMap.LivenessCommit = sec.Key("liveness_commit").MustFloat64(0.2) + RadarMap.LivenessIssue = sec.Key("liveness_issue").MustFloat64(0.2) + RadarMap.LivenessPR = sec.Key("liveness_pr").MustFloat64(0.2) + RadarMap.LivenessRelease = sec.Key("liveness_release").MustFloat64(0.4) + RadarMap.ProjectHealth = sec.Key("project_health").MustFloat64(0.1) + RadarMap.ProjectHealthIssueCompleteRatio = sec.Key("project_health_issue_complete_ratio").MustFloat64(100) + RadarMap.TeamHealth = sec.Key("team_health").MustFloat64(0.1) + RadarMap.TeamHealthContributors = sec.Key("team_health_contributors").MustFloat64(0.2) + RadarMap.TeamHealthKeyContributors = sec.Key("team_health_key_contributors").MustFloat64(0.6) + RadarMap.TeamHealthContributorsAdded = sec.Key("team_health_contributors_added").MustFloat64(0.2) + RadarMap.Growth = sec.Key("growth").MustFloat64(0.1) + RadarMap.GrowthCodeLines = sec.Key("growth_code_lines").MustFloat64(0.2) + RadarMap.GrowthIssue = sec.Key("growth_issue").MustFloat64(0.2) + RadarMap.GrowthContributors = sec.Key("growth_contributors").MustFloat64(0.2) + RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2) + RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2) + } func loadInternalToken(sec *ini.Section) string { diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 77fe49b4e..bd73281d0 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,16 +5,27 @@ package storage import ( - "github.com/unknwon/com" + "io" "path" "strconv" "strings" + "github.com/unknwon/com" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" ) +type FileInfo struct { + FileName string `json:"FileName"` + ModTime string `json:"ModTime"` + IsDir bool `json:"IsDir"` + Size int64 `json:"Size"` + ParenDir string `json:"ParenDir"` + UUID string `json:"UUID"` +} + //check if has the object //todo：修改查询方式 func ObsHasObject(path string) (bool, error) { @@ -102,6 +113,108 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { return nil } +func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error { + input := &obs.UploadPartInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.UploadId = uploadId + input.PartNumber = partNumber + input.Body = putBody + output, err := ObsCli.UploadPart(input) + if err == nil { + log.Info("RequestId:%s\n", output.RequestId) + log.Info("ETag:%s\n", output.ETag) + return nil + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info(obsError.Code) + log.Info(obsError.Message) + return obsError + } else { + log.Error("error:", err.Error()) + return err + } + } + +} + +func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { + input := &obs.GetObjectInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + output, err := ObsCli.GetObject(input) + if err == nil { + log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", + output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) + return output.Body, nil + } else if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + return nil, obsError + } else { + return nil, err + } +} + +func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { + input := &obs.GetObjectInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, JobName, setting.OutPutPath, fileName), "/") + // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + output, err := ObsCli.GetObject(input) + if err == nil { + log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", + output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) + return output.Body, nil + } else if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + return nil, obsError + } else { + return nil, err + } +} + +func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = setting.Bucket + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + if err == nil { + for _, val := range output.Contents { + str1 := strings.Split(val.Key, "/") + var isDir bool + var fileName,nextParentDir string + if strings.HasSuffix(val.Key, "/") { + fileName = str1[len(str1)-2] + isDir = true + nextParentDir = fileName + if fileName == parentDir || (fileName + "/") == setting.OutPutPath { + continue + } + } else { + fileName = str1[len(str1)-1] + isDir = false + } + + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir:isDir, + ParenDir: nextParentDir, + } + fileInfos = append(fileInfos, fileInfo) + } + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } +} + func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} @@ -125,6 +238,26 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file return output.SignedUrl, nil } +func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { + input := &obs.CreateSignedUrlInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") + + input.Expires = 60 * 60 + input.Method = obs.HttpMethodGet + + reqParams := make(map[string]string) + reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" + input.QueryParams = reqParams + output, err := ObsCli.CreateSignedUrl(input) + if err != nil { + log.Error("CreateSignedUrl failed:", err.Error()) + return "", err + } + + return output.SignedUrl, nil +} + func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Method = obs.HttpMethodGet diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 746e46463..7fa8c7a4f 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -755,15 +755,24 @@ unit_disabled = The site administrator has disabled this repository section. language_other = Other datasets = Datasets datasets.desc = Enable Dataset -cloudbrain=cloudbrain + +debug=Debug +stop=Stop +delete=Delete +model_download=Model Download +submit_image=Submit Image +download=Download + + +cloudbrain=Cloudbrain cloudbrain.new=New cloudbrain -cloudbrain.desc=cloudbrain +cloudbrain.desc=Cloudbrain cloudbrain.cancel=Cancel -cloudbrain.commit_image = submit -clone_cnt=download -balance = balance -balance.total_view = total balance -balance.available = available balance： +cloudbrain.commit_image = Submit +clone_cnt=Download +balance = Balance +balance.total_view = Total Balance +balance.available = Available Balance： cloudbrain1 = cloudbrain1 cloudbrain2 = cloudbrain2 cloudbrain_selection = select cloudbrain @@ -774,6 +783,60 @@ cloudbrain_creator=Creator cloudbrain_task = Task Name cloudbrain_operate = Operate cloudbrain_status_createtime = Status/Createtime +cloudbrain_status_runtime = Running Time + + +modelarts.notebook=Debug Task +modelarts.train_job=Train Task +modelarts.train_job.new_debug= New Debug Task +modelarts.train_job.new_train=New Train Task +modelarts.train_job.config=Configuration information +modelarts.train_job.new=New train Task +modelarts.train_job.new_place=The description should not exceed 256 characters + + + +modelarts.train_job.basic_info=Basic Info +modelarts.train_job.job_status=Job Status +modelarts.train_job.job_name=Job Name +modelarts.train_job.version=Job Version +modelarts.train_job.start_time=Start Time +modelarts.train_job.dura_time=Running Time +modelarts.train_job.description=Description +modelarts.train_job.parameter_setting=Parameter setting +modelarts.train_job.parameter_setting_info=Parameter Info +modelarts.train_job.fast_parameter_setting=fast_parameter_setting +modelarts.train_job.fast_parameter_setting_config=fast_parameter_setting_config +modelarts.train_job.fast_parameter_setting_config_link=fast_parameter_setting_config_link +modelarts.train_job.frames=frames +modelarts.train_job.algorithm_origin=Algorithm Origin +modelarts.train_job.AI_driver=AI Engine +modelarts.train_job.start_file=Start File +modelarts.train_job.boot_file_helper=The startup file is the entry file that your program executes, and it must be a file ending in .py +modelarts.train_job.dataset=Dataset +modelarts.train_job.run_parameter=Run Parameter +modelarts.train_job.add_run_parameter=Add Run Parameter +modelarts.train_job.parameter_name=Parameter Name +modelarts.train_job.parameter_value=Parameter Value +modelarts.train_job.resource_setting=resource_setting +modelarts.train_job.resource_setting_info=resource_setting_info +modelarts.train_job.resource_pool=resource_pool +modelarts.train_job.resource_type=resource_type +modelarts.train_job.standard=Standard +modelarts.train_job.NAS_address=NAS Address +modelarts.train_job.NAS_mount_path=NAS Mount Path +modelarts.train_job.query_whether_save_parameter=query_whether_save_parameter +modelarts.train_job.save_helper=save_helper +modelarts.train_job.common_frame=common_frame +modelarts.train_job.amount_of_compute_node=Amount of Compute Node +modelarts.train_job.job_parameter_name=job_parameter_name +modelarts.train_job.parameter_description=parameter_description +modelarts.log=Log +modelarts.version_manage=Version Manage +modelarts.back=Back +modelarts.train_job_para_admin=train_job_para_admin +modelarts.train_job_para.edit=train_job_para.edit +modelarts.train_job_para.connfirm=train_job_para.connfirm template.items = Template Items template.git_content = Git Content (Default Branch) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 6dc0d410c..81b4a8459 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -757,6 +757,14 @@ unit_disabled=站点管理员已禁用此项目单元。 language_other=其它 datasets=数据集 datasets.desc=数据集功能 + +debug=调试 +stop=停止 +delete=删除 +model_download=模型下载 +submit_image=提交镜像 +download=模型下载 + cloudbrain=云脑 cloudbrain.new=新建任务 cloudbrain.desc=云脑功能 @@ -776,6 +784,62 @@ cloudbrain_creator=创建者 cloudbrain_task=任务名称 cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 +cloudbrain_status_runtime = 运行时长 +cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 + +modelarts.notebook=调试任务 +modelarts.train_job=训练任务 +modelarts.train_job.new_debug=新建调试任务 +modelarts.train_job.new_train=新建训练任务 +modelarts.train_job.config=配置信息 +modelarts.train_job.new=新建训练任务 +modelarts.train_job.new_place=描述字数不超过256个字符 + + + +modelarts.train_job.basic_info=基本信息 +modelarts.train_job.job_status=任务状态 +modelarts.train_job.job_name=任务名称 +modelarts.train_job.version=任务版本 +modelarts.train_job.start_time=开始时间 +modelarts.train_job.dura_time=运行时长 +modelarts.train_job.description=任务描述 +modelarts.train_job.parameter_setting=参数设置 +modelarts.train_job.parameter_setting_info=参数信息 +modelarts.train_job.fast_parameter_setting=一键式参数配置 +modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 +modelarts.train_job.fast_parameter_setting_config_link=这里 +modelarts.train_job.frames=常用框架 +modelarts.train_job.algorithm_origin=算法来源 +modelarts.train_job.AI_driver=AI引擎 +modelarts.train_job.start_file=启动文件 +modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。 +modelarts.train_job.boot_file_place=填写启动文件路径，默认为train.py +modelarts.train_job.dataset=数据集 +modelarts.train_job.run_parameter=运行参数 +modelarts.train_job.add_run_parameter=增加运行参数 +modelarts.train_job.parameter_name=参数名 +modelarts.train_job.parameter_value=参数值 +modelarts.train_job.resource_setting=资源设置 +modelarts.train_job.resource_setting_info=资源信息 +modelarts.train_job.resource_pool=资源池 +modelarts.train_job.resource_type=资源类型 +modelarts.train_job.standard=规格 +modelarts.train_job.NAS_address=NAS地址 +modelarts.train_job.NAS_mount_path=NAS挂载路径 +modelarts.train_job.query_whether_save_parameter=保存作业参数 +modelarts.train_job.save_helper=保存当前作业的配置参数，后续您可以使用已保存的配置参数快速创建训练作业。 +modelarts.train_job.common_frame=常用框架 +modelarts.train_job.amount_of_compute_node=计算节点个数 +modelarts.train_job.job_parameter_name=任务参数名称 +modelarts.train_job.parameter_description=任务参数描述 +modelarts.log=日志 +modelarts.version_manage=版本管理 +modelarts.back=返回 +modelarts.train_job_para_admin=任务参数管理 +modelarts.train_job_para.edit=编辑 +modelarts.train_job_para.connfirm=确定 + template.items=模板选项 template.git_content=Git数据(默认分支) diff --git a/public/img/org-jd@2x-80.jpg b/public/img/org-jd@2x-80.jpg new file mode 100644 index 000000000..4c99c8acc Binary files /dev/null and b/public/img/org-jd@2x-80.jpg differ diff --git a/public/self/labelTaskPage.js b/public/self/labelTaskPage.js index a160c3961..a68dfaadc 100644 --- a/public/self/labelTaskPage.js +++ b/public/self/labelTaskPage.js @@ -41,7 +41,7 @@ function setDataSetTask(){ //dislpayUser(); getLabelPropertyTask(); displayLabelPropertyTask(); - + dataset_sele_Change(""); $(".ui.dataset.modal").modal("show"); } @@ -132,6 +132,7 @@ function setPredictTask(){ get_model_list(); displayModelTask(); + dataset_auto_sele_Change(""); $(".ui.predict.modal").modal("show"); } @@ -197,13 +198,13 @@ function sele_export_Change(sele){ function dataset_sele_Change(sele){ var dataset_listName = $('#dataset_list option:selected').text(); console.log("select dataset_list =" + dataset_listName); - $("#datasetlabeltaskname").attr({value:dataset_listName+"-人工标注"}); + $("#datasetlabeltaskname").val(dataset_listName+"-人工标注"); } function dataset_auto_sele_Change(sele){ var dataset_listName = $('#dataset_list_auto option:selected').text(); console.log("select dataset_list_auto =" + dataset_listName); - $("#autolabeltaskname").attr({value:dataset_listName+"-自动标注"}); + $("#autolabeltaskname").val(dataset_listName+"-自动标注"); } @@ -309,11 +310,11 @@ function label_task_create(task_name, relate_task_id, taskType,assign_user_id,la success:function(res){ console.log(res); if(res.code == 0){ - alert("自动标注任务创建成功!"); + alert("标注任务创建成功!"); createsucced = true; } else{ - alert("创建自动标注任务失败，" + res.message); + alert("创建标注任务失败，" + res.message); createsucced = false; } }, diff --git a/public/self/test.js b/public/self/test.js new file mode 100644 index 000000000..2839c76ab --- /dev/null +++ b/public/self/test.js @@ -0,0 +1,28 @@ + +function displayDir(uuid){ + console.log('uuid 1=' + uuid); + + var html="\ + \ + \ + 数据集名称\ + 数据集类型\ + 数据集描述\ + 数据集创建者\ + "; + + for (var i=0;i<1;i++){ + var row = "\ + \ + "+uuid+"\ + " + uuid +"\ + 测试\ + 测试\ + 测试\ + "; + html=html+row; + } + + document.getElementById('dataset-files-table').innerHTML=html; + console.log('uuid 2=' + uuid); +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c6f3ee0ac..9dd773c4d 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -853,7 +853,15 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/:jobid", repo.GetCloudbrainTask) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelarts", func() { - m.Get("/:jobid", repo.GetModelArtsTask) + m.Group("/notebook", func() { + m.Get("/:jobid", repo.GetModelArtsNotebook) + }) + m.Group("/train-job", func() { + m.Group("/:jobid", func() { + m.Get("", repo.GetModelArtsTrainJob) + m.Get("/log", repo.TrainJobGetLog) + }) + }) }, reqRepoReader(models.UnitTypeCloudBrain)) }, repoAssignment()) }) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index f4364f1e4..bfba5236b 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -65,12 +65,15 @@ func GetCloudbrainTask(ctx *context.APIContext) { return } - taskRoles := result.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + job.Status = result.JobStatus.State + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - job.ContainerID = taskRes.TaskStatuses[0].ContainerID - job.Status = taskRes.TaskStatuses[0].State + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State + } if result.JobStatus.State != string(models.JobWaiting) { err = models.UpdateJob(job) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 52b83b446..2e825d8cc 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -11,9 +11,10 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "net/http" + "strconv" ) -func GetModelArtsTask(ctx *context.APIContext) { +func GetModelArtsNotebook(ctx *context.APIContext) { var ( err error ) @@ -43,3 +44,84 @@ func GetModelArtsTask(ctx *context.APIContext) { }) } + +func GetModelArtsTrainJob(ctx *context.APIContext) { + var ( + err error + ) + + jobID := ctx.Params(":jobid") + repoID := ctx.Repo.Repository.ID + job, err := models.GetRepoCloudBrainByJobID(repoID, jobID) + if err != nil { + ctx.NotFound(err) + return + } + result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10)) + if err != nil { + ctx.NotFound(err) + return + } + + job.Status = modelarts.TransTrainJobStatus(result.IntStatus) + job.Duration = result.Duration + job.TrainJobDuration = result.TrainJobDuration + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "JobStatus": job.Status, + "JobDuration": job.Duration, + }) + +} + +func TrainJobGetLog(ctx *context.APIContext) { + var ( + err error + ) + + log.Info("test") + + var jobID = ctx.Params(":jobid") + var logFileName = ctx.Query("file_name") + var baseLine = ctx.Query("base_line") + var order = ctx.Query("order") + + if order != modelarts.OrderDesc && order != modelarts.OrderAsc { + log.Error("order(%s) check failed", order) + ctx.JSON(http.StatusBadRequest, map[string]interface{}{ + "err_msg": "order check failed", + }) + return + } + + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err_msg": "GetCloudbrainByJobID failed", + }) + return + } + + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err_msg": "GetTrainJobLog failed", + }) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "StartLine": result.StartLine, + "EndLine": result.EndLine, + "Content": result.Content, + "Lines": result.Lines, + }) +} diff --git a/routers/home.go b/routers/home.go index 70fea437c..71ba5b6c5 100755 --- a/routers/home.go +++ b/routers/home.go @@ -281,10 +281,10 @@ func ExploreDatasets(ctx *context.Context) { } pager := context.NewPagination(int(count), opts.PageSize, page, 5) + ctx.Data["Keyword"] = opts.Keyword pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["Keyword"] = opts.Keyword ctx.Data["Datasets"] = datasets ctx.Data["Total"] = count ctx.Data["PageIsDatasets"] = true diff --git a/routers/private/tool.go b/routers/private/tool.go index a7a7bee9d..b93f17090 100755 --- a/routers/private/tool.go +++ b/routers/private/tool.go @@ -41,4 +41,6 @@ func UpdateAllRepoCommitCnt(ctx *macaron.Context) { func RepoStatisticManually(ctx *macaron.Context) { date := ctx.Query("date") repo.RepoStatisticDaily(date) + repo.SummaryStatisticDaily(date) + repo.TimingCountDataByDate(date) } diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index a8b2c8fbe..2c082a289 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -262,10 +262,15 @@ func GetAttachment(ctx *context.Context) { return } } else { - url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) - if err != nil { - ctx.ServerError("ObsGetPreSignedUrl", err) - return + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_download?uuid=" + attach.UUID + "&file_name=" + attach.Name + log.Info("return url=" + url) + } else { + url, err = storage.ObsGetPreSignedUrl(attach.UUID, attach.Name) + if err != nil { + ctx.ServerError("ObsGetPreSignedUrl", err) + return + } } } @@ -273,8 +278,21 @@ func GetAttachment(ctx *context.Context) { ctx.ServerError("Update", err) return } + if dataSet != nil { + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + } else { + fr, err := storage.Attachments.Open(attach.RelativePath()) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + if err = ServeData(ctx, attach.Name, fr); err != nil { + ctx.ServerError("ServeData", err) + return + } + } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } else { fr, err := storage.Attachments.Open(attach.RelativePath()) if err != nil { @@ -282,7 +300,6 @@ func GetAttachment(ctx *context.Context) { return } defer fr.Close() - if err = increaseDownloadCount(attach, dataSet); err != nil { ctx.ServerError("Update", err) return @@ -662,6 +679,53 @@ func NewMultipart(ctx *context.Context) { } } +func PutOBSProxyUpload(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Query("uploadId") + partNumber := ctx.QueryInt("partNumber") + fileName := ctx.Query("file_name") + + RequestBody := ctx.Req.Body() + + if RequestBody == nil { + ctx.Error(500, fmt.Sprintf("FormFile: %v", RequestBody)) + return + } + + err := storage.ObsMultiPartUpload(uuid, uploadID, partNumber, fileName, RequestBody.ReadCloser()) + if err != nil { + log.Info("upload error.") + } +} + +func GetOBSProxyDownload(ctx *context.Context) { + uuid := ctx.Query("uuid") + fileName := ctx.Query("file_name") + + body, err := storage.ObsDownload(uuid, fileName) + if err != nil { + log.Info("upload error.") + } else { + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } +} + func GetMultipartUploadUrl(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") @@ -689,10 +753,16 @@ func GetMultipartUploadUrl(ctx *context.Context) { return } } else { - url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) - if err != nil { - ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) - return + if setting.PROXYURL != "" { + url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName + log.Info("return url=" + url) + } else { + url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) + if err != nil { + ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) + return + } + log.Info("url=" + url) } } @@ -847,16 +917,22 @@ func HandleUnDecompressAttachment() { } for _, attach := range attachs { - err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name) - if err != nil { - log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error()) - } else { - attach.DecompressState = models.DecompressStateIng - err = models.UpdateAttachment(attach) + if attach.Type == models.TypeCloudBrainOne { + err = worker.SendDecompressTask(contexExt.Background(), attach.UUID, attach.Name) if err != nil { - log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error()) + log.Error("SendDecompressTask(%s) failed:%s", attach.UUID, err.Error()) + } else { + attach.DecompressState = models.DecompressStateIng + err = models.UpdateAttachment(attach) + if err != nil { + log.Error("UpdateAttachment state(%s) failed:%s", attach.UUID, err.Error()) + } } + } else if attach.Type == models.TypeCloudBrainTwo { + attachjson, _ := json.Marshal(attach) + labelmsg.SendDecompressAttachToLabelOBS(string(attachjson)) } + } return diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 648061171..03fba6cd1 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -40,6 +40,8 @@ var ( categories *models.Categories ) +var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) + // MustEnableDataset check if repository enable internal cb func MustEnableCloudbrain(ctx *context.Context) { if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { @@ -98,15 +100,15 @@ func cutString(str string, lens int) string { func jobNamePrefixValid(s string) string { lowStr := strings.ToLower(s) re := regexp.MustCompile(`[^a-z0-9_\\-]+`) - + removeSpecial := re.ReplaceAllString(lowStr, "") - + re = regexp.MustCompile(`^[_\\-]+`) return re.ReplaceAllString(removeSpecial, "") } -func cloudBrainNewDataPrepare(ctx *context.Context) error{ +func cloudBrainNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true t := time.Now() var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -175,6 +177,9 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error{ ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled + ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath + ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled + return nil } @@ -198,7 +203,12 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId - if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } + + if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) @@ -247,7 +257,12 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue, resourceSpecId) + brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { + downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + } + + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) @@ -273,17 +288,30 @@ func CloudBrainShow(ctx *context.Context) { if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB") - ctx.Data["result"] = jobRes taskRoles := jobRes.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - ctx.Data["taskRes"] = taskRes - task.Status = taskRes.TaskStatuses[0].State - task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - err = models.UpdateJob(task) - if err != nil { - ctx.Data["error"] = err.Error() + if jobRes.JobStatus.State != string(models.JobFailed) { + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + ctx.Data["taskRes"] = taskRes + task.Status = taskRes.TaskStatuses[0].State + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + } + } else { + task.Status = jobRes.JobStatus.State + taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{ + { + State: jobRes.JobStatus.State, + }, + }} + ctx.Data["taskRes"] = taskRes + jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05") + jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05") } + + ctx.Data["result"] = jobRes } ctx.Data["task"] = task @@ -293,6 +321,11 @@ func CloudBrainShow(ctx *context.Context) { func CloudBrainDebug(ctx *context.Context) { var jobID = ctx.Params(":jobid") + if !ctx.IsSigned { + log.Error("the user has not signed in") + ctx.Error(http.StatusForbidden, "","the user has not signed in") + return + } task, err := models.GetCloudbrainByJobID(jobID) if err != nil { ctx.ServerError("GetCloudbrainByJobID failed", err) @@ -305,6 +338,11 @@ func CloudBrainDebug(ctx *context.Context) { func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) { var jobID = ctx.Params(":jobid") + if !ctx.IsSigned { + log.Error("the user has not signed in") + ctx.Error(http.StatusForbidden, "","the user has not signed in") + return + } task, err := models.GetCloudbrainByJobID(jobID) if err != nil { ctx.JSON(200, map[string]string{ @@ -343,7 +381,7 @@ func CloudBrainStop(ctx *context.Context) { return } - if task.Status == string(models.JobStopped) { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) { log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) ctx.ServerError("the job has been stopped", errors.New("the job has been stopped")) return @@ -446,7 +484,7 @@ func CloudBrainDel(ctx *context.Context) { return } - if task.Status != string(models.JobStopped) { + if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) { log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) return @@ -482,7 +520,7 @@ func CloudBrainShowModels(ctx *context.Context) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) @@ -552,7 +590,6 @@ func CloudBrainDownloadModel(ctx *context.Context) { ctx.ServerError("PresignedGetURL", err) return } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } @@ -568,6 +605,8 @@ func GetRate(ctx *context.Context) { ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name) } else if job.JobType == string(models.JobTypeSnn4imagenet) { ctx.Redirect(setting.Snn4imagenetServerHost) + } else if job.JobType == string(models.JobTypeBrainScore) { + ctx.Redirect(setting.BrainScoreServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } @@ -579,9 +618,9 @@ func downloadCode(repo *models.Repository, codePath string) error { return err } - configFile, err := os.OpenFile(codePath + "/.git/config", os.O_RDWR, 0666) + configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666) if err != nil { - log.Error("open file(%s) failed:%v", codePath + "/,git/config", err) + log.Error("open file(%s) failed:%v", codePath+"/,git/config", err) return err } @@ -601,10 +640,10 @@ func downloadCode(repo *models.Repository, codePath string) error { } } - if strings.Contains(line, "url") && strings.Contains(line, ".git"){ + if strings.Contains(line, "url") && strings.Contains(line, ".git") { originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n" if len(line) > len(originUrl) { - originUrl += strings.Repeat( " ", len(line) - len(originUrl)) + originUrl += strings.Repeat(" ", len(line)-len(originUrl)) } bytes := []byte(originUrl) _, err := configFile.WriteAt(bytes, pos) diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 612019d46..406f3dc73 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -21,15 +21,6 @@ const ( tplDirIndex base.TplName = "repo/datasets/dirs/index" ) -type FileInfo struct { - FileName string `json:"FileName"` - ModTime string `json:"ModTime"` - IsDir bool `json:"IsDir"` - Size int64 `json:"Size"` - ParenDir string `json:"ParenDir"` - UUID string `json:"UUID"` -} - type RespGetDirs struct { ResultCode string `json:"resultCode"` FileInfos string `json:"fileInfos"` @@ -59,7 +50,7 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:", err.Error()) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 080c36377..f33d345cd 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1,10 +1,13 @@ package repo import ( - "code.gitea.io/gitea/modules/modelarts" "encoding/json" "errors" - "github.com/unknwon/com" + "io" + "io/ioutil" + "net/http" + "os" + "path" "strconv" "strings" "time" @@ -13,14 +16,30 @@ import ( "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + + "github.com/unknwon/com" ) const ( + // tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index" + tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index" + tplModelArtsNotebookNew base.TplName = "repo/modelarts/notebook/new" + tplModelArtsNotebookShow base.TplName = "repo/modelarts/notebook/show" + tplModelArtsIndex base.TplName = "repo/modelarts/index" tplModelArtsNew base.TplName = "repo/modelarts/new" tplModelArtsShow base.TplName = "repo/modelarts/show" + + tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" + tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" + tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" + tplModelArtsTrainJobShowModels base.TplName = "repo/modelarts/trainjob/models/index" ) // MustEnableDataset check if repository enable internal cb @@ -30,6 +49,7 @@ func MustEnableModelArts(ctx *context.Context) { return } } + func ModelArtsIndex(ctx *context.Context) { MustEnableModelArts(ctx) repo := ctx.Repo.Repository @@ -100,7 +120,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) { uuid := form.Attachment description := form.Description //repo := ctx.Repo.Repository - + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } err := modelarts.GenerateTask(ctx, jobName, uuid, description) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form) @@ -228,7 +251,7 @@ func ModelArtsDel(ctx *context.Context) { return } - if task.Status != string(models.JobStopped) { + if task.Status != string(models.ModelArtsCreateFailed) && task.Status != string(models.ModelArtsStartFailed) && task.Status != string(models.ModelArtsStopped) { log.Error("the job(%s) has not been stopped", task.JobName) ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) return @@ -249,3 +272,874 @@ func ModelArtsDel(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts") } + +func NotebookIndex(ctx *context.Context) { + MustEnableModelArts(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + JobType: string(models.JobTypeDebug), + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + for i, task := range ciTasks { + if task.Status == string(models.JobRunning) { + ciTasks[i].CanDebug = true + } else { + ciTasks[i].CanDebug = false + } + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = ciTasks + ctx.HTML(200, tplModelArtsNotebookIndex) +} + +func NotebookNew(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + t := time.Now() + var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return + } + + ctx.Data["attachments"] = attachs + ctx.Data["dataset_path"] = modelarts.DataSetMountPath + ctx.Data["env"] = modelarts.NotebookEnv + ctx.Data["notebook_type"] = modelarts.NotebookType + if modelarts.FlavorInfos == nil { + json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) + } + ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo + + ctx.HTML(200, tplModelArtsNotebookNew) +} + +func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) { + ctx.Data["PageIsNotebook"] = true + jobName := form.JobName + uuid := form.Attachment + description := form.Description + + err := modelarts.GenerateTask(ctx, jobName, uuid, description) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") +} + +func NotebookShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) + return + } + + result, err := modelarts.GetJob(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) + return + } + + if result != nil { + task.Status = result.Status + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookShow, nil) + return + } + + createTime, _ := com.StrTo(result.CreationTimestamp).Int64() + result.CreateTime = time.Unix(int64(createTime/1000), 0).Format("2006-01-02 15:04:05") + endTime, _ := com.StrTo(result.LatestUpdateTimestamp).Int64() + result.LatestUpdateTime = time.Unix(int64(endTime/1000), 0).Format("2006-01-02 15:04:05") + result.QueuingInfo.BeginTime = time.Unix(int64(result.QueuingInfo.BeginTimestamp/1000), 0).Format("2006-01-02 15:04:05") + result.QueuingInfo.EndTime = time.Unix(int64(result.QueuingInfo.EndTimestamp/1000), 0).Format("2006-01-02 15:04:05") + } + + ctx.Data["task"] = task + ctx.Data["jobID"] = jobID + ctx.Data["result"] = result + ctx.HTML(200, tplModelArtsNotebookShow) +} + +func NotebookDebug(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + _, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + result, err := modelarts.GetJob(jobID) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) + return + } + + res, err := modelarts.GetJobToken(jobID) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) + return + } + + urls := strings.Split(result.Spec.Annotations.Url, "/") + urlPrefix := result.Spec.Annotations.TargetDomain + for i, url := range urls { + if i > 2 { + urlPrefix += "/" + url + } + } + + debugUrl := urlPrefix + "?token=" + res.Token + ctx.Redirect(debugUrl) +} + +func NotebookStop(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + log.Info(jobID) + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobRunning) { + log.Error("the job(%s) is not running", task.JobName) + ctx.ServerError("the job is not running", errors.New("the job is not running")) + return + } + + param := models.NotebookAction{ + Action: models.ActionStop, + } + res, err := modelarts.StopJob(jobID, param) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("StopJob failed", err) + return + } + + task.Status = res.CurrentStatus + err = models.UpdateJob(task) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") +} + +func NotebookDel(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobStopped) { + log.Error("the job(%s) has not been stopped", task.JobName) + ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) + return + } + + _, err = modelarts.DelNotebook(jobID) + if err != nil { + log.Error("DelJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("DelJob failed", err) + return + } + + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") +} + +func TrainJobIndex(ctx *context.Context) { + MustEnableModelArts(ctx) + + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //ctx.Data["CanCreate"] = can + + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + JobType: string(models.JobTypeTrain), + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = tasks + ctx.HTML(200, tplModelArtsTrainJobIndex) +} + +func TrainJobNew(ctx *context.Context) { + err := trainJobNewDataPrepare(ctx) + if err != nil { + ctx.ServerError("get new train-job info failed", err) + return + } + ctx.HTML(200, tplModelArtsTrainJobNew) +} + +func trainJobNewDataPrepare(ctx *context.Context) error { + ctx.Data["PageIsCloudBrain"] = true + + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) + // return + //} + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return err + } + ctx.Data["attachments"] = attachs + + var resourcePools modelarts.ResourcePool + if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["engine_versions"] = versionInfos.Version + + var flavorInfos modelarts.Flavor + if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + return err + } + ctx.Data["flavor_infos"] = flavorInfos.Info + + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + ctx.Data["train_url"] = outputObsPath + + configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) + if err != nil { + ctx.ServerError("getConfigList failed:", err) + return err + } + + ctx.Data["config_list"] = configList.ParaConfigs + + return nil +} + +func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { + ctx.Data["PageIsTrainJob"] = true + jobName := form.JobName + uuid := form.Attachment + description := form.Description + workServerNumber := form.WorkServerNumber + engineID := form.EngineID + bootFile := form.BootFile + flavorCode := form.Flavor + params := form.Params + poolID := form.PoolID + isSaveParam := form.IsSaveParam + repo := ctx.Repo.Repository + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" + + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.RenderWithErr("the user can not create train-job", tplModelArtsTrainJobNew, &form) + // return + //} + + //param check + if err := paramCheckCreateTrainJob(form); err != nil { + log.Error("paramCheckCreateTrainJob failed:(%v)", err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) + return + } + + attach, err := models.GetAttachmentByUUID(uuid) + if err != nil { + log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error()) + return + } + + //todo: del the codeLocalPath + _, err = ioutil.ReadDir(codeLocalPath) + if err == nil { + os.RemoveAll(codeLocalPath) + } + if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{}); err != nil { + log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err) + trainJobNewDataPrepare(ctx) + + ctx.Data["bootFile"] = form.BootFile + ctx.Data["uuid"] = form.Attachment + ctx.Data["datasetName"] = attach.Name + ctx.Data["params"] = form.Params + trainJobNewDataPrepare(ctx) + // ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobNew, &form) + // ctx.RenderWithErr(err, tplModelArtsTrainJobNew, &form) + return + } + + //todo: upload code (send to file_server todo this work?) + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { + log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form) + return + } + + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath); err != nil { + log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form) + return + } + + if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form) + return + } + + //todo: del local code? + + var parameters models.Parameters + param := make([]models.Parameter, 0) + param = append(param, models.Parameter{ + Label: modelarts.TrainUrl, + Value: outputObsPath, + }, models.Parameter{ + Label: modelarts.DataUrl, + Value: dataPath, + }) + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobNew, &form) + return + } + + for _, parameter := range parameters.Parameter { + if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl { + param = append(param, models.Parameter{ + Label: parameter.Label, + Value: parameter.Value, + }) + } + } + } + + //save param config + if isSaveParam == "on" { + if form.ParameterTemplateName == "" { + log.Error("ParameterTemplateName is empty") + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("保存作业参数时，作业参数名称不能为空", tplModelArtsTrainJobNew, &form) + return + } + + _, err := modelarts.CreateTrainJobConfig(models.CreateConfigParams{ + ConfigName: form.ParameterTemplateName, + Description: form.PrameterDescription, + DataUrl: dataPath, + AppUrl: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + TrainUrl: outputObsPath, + Flavor: models.Flavor{ + Code: flavorCode, + }, + WorkServerNum: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Parameter: param, + }) + + if err != nil { + log.Error("Failed to CreateTrainJobConfig: %v", err) + trainJobNewDataPrepare(ctx) + ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form) + return + } + } + + req := &modelarts.GenerateTrainJobReq{ + JobName: jobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFile: codeObsPath + bootFile, + TrainUrl: outputObsPath, + FlavorCode: flavorCode, + WorkServerNumber: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Parameters: param, + } + + err = modelarts.GenerateTrainJob(ctx, req) + if err != nil { + log.Error("GenerateTrainJob failed:%v", err.Error()) + trainJobNewDataPrepare(ctx) + ctx.Data["bootFile"] = form.BootFile + ctx.Data["uuid"] = form.Attachment + ctx.Data["datasetName"] = attach.Name + ctx.Data["params"] = form.Params + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") +} + +// readDir reads the directory named by dirname and returns +// a list of directory entries sorted by filename. +func readDir(dirname string) ([]os.FileInfo, error) { + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + + list, err := f.Readdir(100) + f.Close() + if err != nil { + //todo: can not upload empty folder + if err == io.EOF { + return nil, nil + } + return nil, err + } + + //sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) + return list, nil +} + +func uploadCodeToObs(codePath, jobName, parentDir string) error { + files, err := readDir(codePath) + if err != nil { + log.Error("readDir(%s) failed: %s", codePath, err.Error()) + return err + } + + for _, file := range files { + if file.IsDir() { + input := &obs.PutObjectInput{} + input.Bucket = setting.Bucket + input.Key = parentDir + file.Name() + "/" + _, err = storage.ObsCli.PutObject(input) + if err != nil { + log.Error("PutObject(%s) failed: %s", input.Key, err.Error()) + return err + } + + if err = uploadCodeToObs(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil { + log.Error("uploadCodeToObs(%s) failed: %s", file.Name(), err.Error()) + return err + } + } else { + input := &obs.PutFileInput{} + input.Bucket = setting.Bucket + input.Key = setting.CodePathPrefix + jobName + "/code/" + parentDir + file.Name() + input.SourceFile = codePath + file.Name() + _, err = storage.ObsCli.PutFile(input) + if err != nil { + log.Error("PutFile(%s) failed: %s", input.SourceFile, err.Error()) + return err + } + } + } + + return nil +} + +func obsMkdir(dir string) error { + input := &obs.PutObjectInput{} + input.Bucket = setting.Bucket + input.Key = dir + _, err := storage.ObsCli.PutObject(input) + if err != nil { + log.Error("PutObject(%s) failed: %s", input.Key, err.Error()) + return err + } + + return nil +} + +func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error { + if !strings.HasSuffix(form.BootFile, ".py") { + log.Error("the boot file(%s) must be a python file", form.BootFile) + return errors.New("启动文件必须是python文件") + } + + if form.WorkServerNumber > 25 || form.WorkServerNumber < 1 { + log.Error("the WorkServerNumber(%d) must be in (1,25)", form.WorkServerNumber) + return errors.New("计算节点数必须在1-25之间") + } + + return nil +} + +func TrainJobShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + // attach, err := models.GetAttachmentByUUID(task.Uuid) + // if err != nil { + // log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error()) + // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + // return + // } + + result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetJob(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + if result != nil { + result.CreateTime = time.Unix(int64(result.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05") + if result.Duration != 0 { + result.TrainJobDuration = addZero(result.Duration/3600000) + ":" + addZero(result.Duration%3600000/60000) + ":" + addZero(result.Duration%60000/1000) + + } else { + result.TrainJobDuration = "00:00:00" + } + result.Status = modelarts.TransTrainJobStatus(result.IntStatus) + err = models.SetTrainJobStatusByJobID(jobID, result.Status, result.Duration, string(result.TrainJobDuration)) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } + + result.DatasetName = task.DatasetName + } + + resultLogFile, resultLog, err := trainJobGetLog(jobID) + if err != nil { + log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] + ctx.Data["log"] = resultLog + ctx.Data["task"] = task + ctx.Data["jobID"] = jobID + ctx.Data["result"] = result + ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) +} + +func addZero(t int64) (m string) { + if t < 10 { + m = "0" + strconv.FormatInt(t, 10) + return m + } else { + return strconv.FormatInt(t, 10) + } +} + +func TrainJobGetLog(ctx *context.Context) { + ctx.Data["PageIsTrainJob"] = true + + var jobID = ctx.Params(":jobid") + var logFileName = ctx.Query("file_name") + var baseLine = ctx.Query("base_line") + var order = ctx.Query("order") + + if order != modelarts.OrderDesc && order != modelarts.OrderAsc { + log.Error("order(%s) check failed", order) + ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow) + return + } + + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + ctx.Data["log"] = result + //ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) +} + +func trainJobGetLog(jobID string) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) { + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + return nil, nil, err + } + + resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error()) + return nil, nil, err + } + + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), "", resultLogFile.LogFileList[0], modelarts.OrderDesc, modelarts.Lines) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) + return nil, nil, err + } + + return resultLogFile, result, err +} + +func TrainJobDel(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) + return + } + + _, err = modelarts.DelTrainJob(jobID) + if err != nil { + log.Error("DelTrainJob(%s) failed:%v", task.JobName, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) + return + } + + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") +} + +func TrainJobStop(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) + return + } + + _, err = modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("StopTrainJob(%s) failed:%v", task.JobName, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") +} + +func canUserCreateTrainJob(uid int64) (bool, error) { + org, err := models.GetOrgByName(setting.AllowedOrg) + if err != nil { + log.Error("get allowed org failed: ", setting.AllowedOrg) + return false, err + } + + return org.IsOrgMember(uid) +} + +func TrainJobGetConfigList(ctx *context.Context) { + ctx.Data["PageIsTrainJob"] = true + + var jobID = ctx.Params(":jobid") + var logFileName = ctx.Query("file_name") + var baseLine = ctx.Query("base_line") + var order = ctx.Query("order") + + if order != modelarts.OrderDesc && order != modelarts.OrderAsc { + log.Error("order(%s) check failed", order) + ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow) + return + } + + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines) + if err != nil { + log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) + return + } + + ctx.Data["log"] = result + //ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) +} + +func getConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) { + var result models.GetConfigListResult + + list, err := modelarts.GetConfigList(perPage, page, sortBy, order, searchContent, configType) + if err != nil { + log.Error("GetConfigList failed:", err) + return &result, err + } + + for _, config := range list.ParaConfigs { + paraConfig, err := modelarts.GetParaConfig(config.ConfigName, configType) + if err != nil { + log.Error("GetParaConfig failed:", err) + return &result, err + } + + config.Result = paraConfig + } + + return list, nil +} + +func TrainJobShowModels(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + jobID := ctx.Params(":jobid") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("no such job!", ctx.Data["msgID"]) + ctx.ServerError("no such job:", err) + return + } + + models, err := storage.GetObsListObject(task.JobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.Data["Path"] = dirArray + ctx.Data["Dirs"] = models + ctx.Data["task"] = task + ctx.Data["JobID"] = jobID + ctx.HTML(200, tplModelArtsTrainJobShowModels) +} + +func TrainJobDownloadModel(ctx *context.Context) { + parentDir := ctx.Query("parentDir") + fileName := ctx.Query("fileName") + jobName := ctx.Query("jobName") + url, err := storage.GetObsCreateSignedUrl(jobName, parentDir, fileName) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +} diff --git a/routers/repo/repo.go b/routers/repo/repo.go index b0bb608d0..437521d5a 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -531,6 +531,8 @@ func Download(ctx *context.Context) { } } + ctx.Repo.Repository.IncreaseCloneCnt() + ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 38db0f5cb..2de22a1ee 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -3,6 +3,10 @@ package repo import ( "time" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/normalization" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" @@ -12,11 +16,14 @@ import ( func RepoStatisticAuto() { log.Info("", time.Now()) yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + setting.UpdateRadarMap() RepoStatisticDaily(yesterday) } func RepoStatisticDaily(date string) { log.Info("%s", date) + log.Info("begin Repo Statistic") + t, _ := time.Parse("2006-01-02", date) if err := models.DeleteRepoStatDaily(date); err != nil { log.Error("DeleteRepoStatDaily failed: %v", err.Error()) return @@ -28,9 +35,14 @@ func RepoStatisticDaily(date string) { return } - for _, repo := range repos { + var reposRadar = make([]*models.RepoStatistic, 0) + + var minRepoRadar models.RepoStatistic + var maxRepoRadar models.RepoStatistic + + for i, repo := range repos { log.Info("start statistic: %s", repo.Name) - var numDevMonths,numWikiViews,numContributor,numKeyContributor int64 + var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64 repoGitStat, err := models.GetRepoKPIStats(repo) if err != nil { log.Error("GetRepoKPIStats failed: %s", repo.Name) @@ -39,6 +51,9 @@ func RepoStatisticDaily(date string) { numKeyContributor = repoGitStat.KeyContributors numWikiViews = repoGitStat.WikiPages numContributor = repoGitStat.Contributors + numCommitsGrowth = repoGitStat.CommitsAdded + numCommitLinesGrowth = repoGitStat.CommitLinesModified + numContributorsGrowth = repoGitStat.ContributorsAdded } var issueFixedRate float32 @@ -46,7 +61,7 @@ func RepoStatisticDaily(date string) { issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues) } - var numVersions int64 + var numVersions int64 numVersions, err = models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{}) if err != nil { log.Error("GetReleaseCountByRepoID failed(%s): %v", repo.Name, err) @@ -72,26 +87,60 @@ func RepoStatisticDaily(date string) { } repoStat := models.RepoStatistic{ - RepoID: repo.ID, - Date: date, - NumWatches: int64(repo.NumWatches), - NumStars: int64(repo.NumStars), - NumDownloads: repo.CloneCnt, - NumComments: numComments, - NumVisits: int64(numVisits), - NumClosedIssues: int64(repo.NumClosedIssues), - NumVersions: numVersions, - NumDevMonths: numDevMonths, - RepoSize: repo.Size, - DatasetSize: datasetSize, - NumModels: 0, - NumWikiViews: numWikiViews, - NumCommits: repo.NumCommit, - NumIssues: int64(repo.NumIssues), - NumPulls: int64(repo.NumPulls), - IssueFixedRate: issueFixedRate, - NumContributor: numContributor, - NumKeyContributor: numKeyContributor, + RepoID: repo.ID, + Date: date, + Name: repo.Name, + IsPrivate: repo.IsPrivate, + NumWatches: int64(repo.NumWatches), + NumStars: int64(repo.NumStars), + NumDownloads: repo.CloneCnt, + NumComments: numComments, + NumVisits: int64(numVisits), + NumClosedIssues: int64(repo.NumClosedIssues), + NumVersions: numVersions, + NumDevMonths: numDevMonths, + RepoSize: repo.Size, + DatasetSize: datasetSize, + NumModels: 0, + NumWikiViews: numWikiViews, + NumCommits: repo.NumCommit, + NumIssues: int64(repo.NumIssues), + NumPulls: int64(repo.NumPulls), + IssueFixedRate: issueFixedRate, + NumContributor: numContributor, + NumKeyContributor: numKeyContributor, + NumCommitsGrowth: numCommitsGrowth, + NumCommitLinesGrowth: numCommitLinesGrowth, + NumContributorsGrowth: numContributorsGrowth, + } + + dayBeforeDate := t.AddDate(0, 0, -1).Format("2006-01-02") + repoStatisticsBefore, err := models.GetRepoStatisticByDate(dayBeforeDate) + + if err != nil { + log.Error("get data of day before the date failed ", err) + } else { + if len(repoStatisticsBefore) > 0 { + repoStatisticBefore := repoStatisticsBefore[0] + repoStat.NumWatchesAdded = repoStat.NumWatches - repoStatisticBefore.NumWatches + repoStat.NumStarsAdded = repoStat.NumStars - repoStatisticBefore.NumStars + repoStat.NumForksAdded = repoStat.NumForks - repoStatisticBefore.NumForks + repoStat.NumDownloadsAdded = repoStat.NumDownloads - repoStatisticBefore.NumDownloads + repoStat.NumCommentsAdded = repoStat.NumComments - repoStatisticBefore.NumComments + repoStat.NumClosedIssuesAdded = repoStat.NumClosedIssues - repoStatisticBefore.NumClosedIssues + repoStat.NumCommitsAdded = repoStat.NumCommits - repoStatisticBefore.NumCommits + repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues + repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls + repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor + } + } + day4MonthsAgo := t.AddDate(0, -4, 0) + repoStatisticFourMonthsAgo, err := models.GetOneRepoStatisticBeforeTime(day4MonthsAgo) + if err != nil { + log.Error("Get data of 4 moth ago failed.", err) + } else { + repoStat.NumCommentsGrowth = repoStat.NumComments - repoStatisticFourMonthsAgo.NumComments + repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues } if _, err = models.InsertRepoStat(&repoStat); err != nil { @@ -100,9 +149,92 @@ func RepoStatisticDaily(date string) { continue } + tempRepoStat := models.RepoStatistic{ + RepoID: repoStat.RepoID, + Date: repoStat.Date, + Impact: normalization.GetImpactInitValue(repoStat.NumWatches, repoStat.NumStars, repoStat.NumForks, repoStat.NumDownloads, repoStat.NumComments, repoStat.NumVisits), + Completeness: normalization.GetCompleteInitValue(repoStat.NumClosedIssues, repoStat.NumVersions, repoStat.NumDevMonths, repoStat.DatasetSize, repoStat.NumModels, repoStat.NumWikiViews), + Liveness: normalization.GetLivenessInitValue(repoStat.NumCommits, repoStat.NumIssues, repoStat.NumPulls, repoStat.NumVisits), + ProjectHealth: normalization.GetProjectHealthInitValue(repoStat.IssueFixedRate), + TeamHealth: normalization.GetTeamHealthInitValue(repoStat.NumContributor, repoStat.NumKeyContributor, repoStat.NumContributorsGrowth), + Growth: normalization.GetRepoGrowthInitValue(repoStat.NumCommitLinesGrowth, repoStat.NumIssuesGrowth, repoStat.NumCommitsGrowth, repoStat.NumContributorsGrowth, repoStat.NumCommentsGrowth), + } + + reposRadar = append(reposRadar, &tempRepoStat) + + if i == 0 { + minRepoRadar = tempRepoStat + maxRepoRadar = tempRepoStat + } else { + + if tempRepoStat.Impact < minRepoRadar.Impact { + minRepoRadar.Impact = tempRepoStat.Impact + } + + if tempRepoStat.Impact > maxRepoRadar.Impact { + maxRepoRadar.Impact = tempRepoStat.Impact + } + + if tempRepoStat.Completeness < minRepoRadar.Completeness { + minRepoRadar.Completeness = tempRepoStat.Completeness + } + + if tempRepoStat.Completeness > maxRepoRadar.Completeness { + maxRepoRadar.Completeness = tempRepoStat.Completeness + } + + if tempRepoStat.Liveness < minRepoRadar.Completeness { + minRepoRadar.Liveness = tempRepoStat.Liveness + } + + if tempRepoStat.Liveness > maxRepoRadar.Liveness { + maxRepoRadar.Liveness = tempRepoStat.Liveness + } + + if tempRepoStat.ProjectHealth < minRepoRadar.ProjectHealth { + minRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth + } + + if tempRepoStat.ProjectHealth > maxRepoRadar.ProjectHealth { + maxRepoRadar.ProjectHealth = tempRepoStat.ProjectHealth + } + + if tempRepoStat.TeamHealth < minRepoRadar.TeamHealth { + minRepoRadar.TeamHealth = tempRepoStat.TeamHealth + } + + if tempRepoStat.TeamHealth > maxRepoRadar.TeamHealth { + maxRepoRadar.TeamHealth = tempRepoStat.TeamHealth + } + + if tempRepoStat.Growth < minRepoRadar.Growth { + minRepoRadar.Growth = tempRepoStat.Growth + } + + if tempRepoStat.Growth > maxRepoRadar.Growth { + maxRepoRadar.Growth = tempRepoStat.Growth + } + + } + log.Info("finish statistic: %s", repo.Name) } + //radar map + log.Info("begin statistic radar") + for _, radarInit := range reposRadar { + radarInit.Impact = normalization.Normalization(radarInit.Impact, minRepoRadar.Impact, maxRepoRadar.Impact) + radarInit.Completeness = normalization.Normalization(radarInit.Completeness, minRepoRadar.Completeness, maxRepoRadar.Completeness) + radarInit.Liveness = normalization.Normalization(radarInit.Liveness, minRepoRadar.Liveness, maxRepoRadar.Liveness) + radarInit.ProjectHealth = normalization.Normalization(radarInit.ProjectHealth, minRepoRadar.ProjectHealth, maxRepoRadar.ProjectHealth) + radarInit.TeamHealth = normalization.Normalization(radarInit.TeamHealth, minRepoRadar.TeamHealth, maxRepoRadar.TeamHealth) + radarInit.Growth = normalization.Normalization(radarInit.Growth, minRepoRadar.Growth, maxRepoRadar.Growth) + radarInit.RadarTotal = normalization.GetRadarValue(radarInit.Impact, radarInit.Completeness, radarInit.Liveness, radarInit.ProjectHealth, radarInit.TeamHealth, radarInit.Growth) + models.UpdateRepoStat(radarInit) + } + + log.Info("finish statistic: radar") + } func getDatasetSize(repo *models.Repository) (int64, error) { diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go new file mode 100644 index 000000000..53270664c --- /dev/null +++ b/routers/repo/repo_summary_statistic.go @@ -0,0 +1,94 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +func SummaryStatistic() { + log.Info("Generate summary statistic begin") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + SummaryStatisticDaily(yesterday) + log.Info("Generate summary statistic end") +} + +func SummaryStatisticDaily(date string) { + log.Info("%s", date) + if err := models.DeleteSummaryStatisticDaily(date); err != nil { + log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + return + } + + //user number + userNumber, err := models.GetUsersCount() + if err != nil { + log.Error("can not get user number", err) + userNumber = 0 + } + //organization number + organizationNumber, err := models.GetOrganizationsCount() + if err != nil { + log.Error("can not get orgnazition number", err) + organizationNumber = 0 + } + // repository number + repositoryNumer, err := models.GetAllRepositoriesCount() + if err != nil { + log.Error("can not get repository number", err) + repositoryNumer = 0 + } + //repository size + repositorySize, err := models.GetAllRepositoriesSize() + if err != nil { + log.Error("can not get repository size", err) + repositorySize = 0 + } + // dataset size + allDatasetSize, err := models.GetAllAttachmentSize() + if err != nil { + log.Error("can not get dataset size", err) + allDatasetSize = 0 + } + //topic repo number + topics, err := models.GetAllUsedTopics() + if err != nil { + log.Error("can not get topics", err) + } + var topicsCount [11]int + for _, topic := range topics { + + index, exists := models.DomainMap[topic.Name] + if exists { + topicsCount[index] = topic.RepoCount + } + + } + + summaryStat := models.SummaryStatistic{ + Date: date, + NumUsers: userNumber, + RepoSize: repositorySize, + DatasetSize: allDatasetSize, + NumOrganizations: organizationNumber, + NumRepos: repositoryNumer, + NumRepoBigModel: topicsCount[0], + NumRepoAI: topicsCount[1], + NumRepoVision: topicsCount[2], + NumRepoNLP: topicsCount[3], + NumRepoML: topicsCount[4], + NumRepoNN: topicsCount[5], + NumRepoAutoDrive: topicsCount[6], + NumRepoRobot: topicsCount[7], + NumRepoLeagueLearn: topicsCount[8], + NumRepoDataMining: topicsCount[9], + NumRepoRISC: topicsCount[10], + } + + if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil { + log.Error("Insert summary Stat failed: %v", err.Error()) + } + + log.Info("finish summary statistic") +} diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 3260780ac..7dc7af321 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -8,16 +8,17 @@ import ( "code.gitea.io/gitea/modules/log" ) -func TimingCountData() { +func TimingCountDataByDate(date string) { + + t, _ := time.Parse("2006-01-02", date) + startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + + endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location()) + //query wiki data log.Info("start to time count data") wikiMap := make(map[string]int) - currentTimeNow := time.Now() - log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - - yesterday := currentTimeNow.AddDate(0, 0, -1) - repoList, err := models.GetAllRepositories() if err != nil { log.Error("query repo error.") @@ -29,7 +30,7 @@ func TimingCountData() { time, err := git.GetLatestCommitTime(wikiPath) if err == nil { log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath) - if time.After(yesterday) { + if time.After(startTime) { wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath) if err != nil { log.Error("wiki not exist. wikiPath=" + wikiPath) @@ -55,5 +56,16 @@ func TimingCountData() { } } //other user info data - models.CountData(wikiMap) + models.CounDataByDate(wikiMap, startTime, endTime) + +} + +func TimingCountData() { + + log.Info("start to time count data") + currentTimeNow := time.Now() + log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) + startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02") + + TimingCountDataByDate(startTime) } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 892368315..7e7d0642a 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,13 +6,15 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/operation" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/operation" + "code.gitea.io/gitea/routers/private" + "code.gitea.io/gitea/routers/secure" "code.gitea.io/gitea/models" @@ -33,7 +35,6 @@ import ( "code.gitea.io/gitea/routers/dev" "code.gitea.io/gitea/routers/events" "code.gitea.io/gitea/routers/org" - "code.gitea.io/gitea/routers/private" "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/routers/user" userSetting "code.gitea.io/gitea/routers/user/setting" @@ -113,14 +114,14 @@ func RouterHandler(level log.Level) func(ctx *macaron.Context) { } // SetLogMsgID set msgID in Context -func SetLogMsgID() func(ctx *macaron.Context) { +func SetLogMsgID() macaron.Handler { return func(ctx *macaron.Context) { start := time.Now() uuid := gouuid.NewV4().String() ctx.Data["MsgID"] = uuid - log.Info("Started %s %s for %s", log.ColoredMethod(ctx.Req.Method), ctx.Req.URL.RequestURI(), ctx.RemoteAddr(), ctx.Data["MsgID"]) + log.Info("%s Started %s %s for %s", ctx.Data["SignedUserName"], log.ColoredMethod(ctx.Req.Method), ctx.Req.URL.RequestURI(), ctx.RemoteAddr(), ctx.Data["MsgID"]) rw := ctx.Resp.(macaron.ResponseWriter) ctx.Next() @@ -148,7 +149,7 @@ func NewMacaron() *macaron.Macaron { m.Use(macaron.Logger()) } } - m.Use(SetLogMsgID()) + //m.Use(SetLogMsgID()) // Access Logger is similar to Router Log but more configurable and by default is more like the NCSA Common Log format if setting.EnableAccessLog { setupAccessLogger(m) @@ -256,6 +257,7 @@ func NewMacaron() *macaron.Macaron { DisableDebug: !setting.EnablePprof, })) m.Use(context.Contexter()) + m.Use(SetLogMsgID()) // OK we are now set-up enough to allow us to create a nicer recovery than // the default macaron recovery m.Use(context.Recovery()) @@ -565,6 +567,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/private", repo.UpdatePublicAttachment) m.Get("/get_chunks", repo.GetSuccessChunks) m.Get("/new_multipart", repo.NewMultipart) + m.Put("/obs_proxy_multipart", repo.PutOBSProxyUpload) + m.Get("/obs_proxy_download", repo.GetOBSProxyDownload) m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) m.Post("/complete_multipart", repo.CompleteMultipart) m.Post("/update_chunk", repo.UpdateMultipart) @@ -958,15 +962,42 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelarts", func() { - m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex) - m.Group("/:jobid", func() { - m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow) - m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug) - m.Post("/stop", reqRepoCloudBrainWriter, repo.ModelArtsStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.ModelArtsDel) + // m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex) + // m.Group("/:jobid", func() { + // m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow) + // m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug) + // m.Post("/stop", reqRepoCloudBrainWriter, repo.ModelArtsStop) + // m.Post("/del", reqRepoCloudBrainWriter, repo.ModelArtsDel) + // }) + // m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew) + // m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate) + + m.Group("/notebook", func() { + m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) + m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) + m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) + m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) + }) + m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) + }) + + m.Group("/train-job", func() { + m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) + m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) + m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) + m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog) + m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobShowModels) + m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel) + }) + m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) + m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) }) - m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew) - m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate) }, context.RepoRef()) m.Group("/blockchain", func() { diff --git a/routers/user/auth.go b/routers/user/auth.go index 126d0a4c8..44c5ad97d 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -11,8 +11,6 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/routers/repo" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/auth/oauth2" @@ -217,6 +215,7 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { } return } + models.SaveLoginInfoToDb(ctx.Req.Request, u) // If this user is enrolled in 2FA, we can't sign the user in just yet. // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) @@ -228,7 +227,6 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { } return } - // User needs to use 2FA, save data and redirect to 2FA page. if err := ctx.Session.Set("twofaUid", u.ID); err != nil { ctx.ServerError("UserSignIn: Unable to set twofaUid in session", err) @@ -242,7 +240,6 @@ func SignInPost(ctx *context.Context, form auth.SignInForm) { ctx.ServerError("UserSignIn: Unable to save session", err) return } - regs, err := models.GetU2FRegistrationsByUID(u.ID) if err == nil && len(regs) > 0 { ctx.Redirect(setting.AppSubURL + "/user/u2f") @@ -1058,7 +1055,6 @@ func SignOut(ctx *context.Context) { }) } HandleSignOut(ctx) - go repo.StopJobsByUserID(ctx.User.ID) ctx.Redirect(setting.AppSubURL + "/") } @@ -1171,8 +1167,8 @@ func SignUpPost(ctx *context.Context, cpt *captcha.Captcha, form auth.RegisterFo log.Trace("Account created: %s", u.Name, ctx.Data["MsgID"]) err := models.AddEmailAddress(&models.EmailAddress{ - UID: u.ID, - Email: form.Email, + UID: u.ID, + Email: form.Email, IsActivated: !setting.Service.RegisterEmailConfirm, }) @@ -1270,7 +1266,7 @@ func Activate(ctx *context.Context) { } email, err := models.GetEmailAddressByIDAndEmail(user.ID, user.Email) - if err != nil || email == nil{ + if err != nil || email == nil { log.Error("GetEmailAddressByIDAndEmail failed", ctx.Data["MsgID"]) } else { if err := email.Activate(); err != nil { diff --git a/routers/user/setting/account.go b/routers/user/setting/account.go index 6165bfc5b..0a0fc558c 100644 --- a/routers/user/setting/account.go +++ b/routers/user/setting/account.go @@ -8,6 +8,8 @@ package setting import ( "errors" + "code.gitea.io/gitea/routers/repo" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -240,6 +242,7 @@ func DeleteAccount(ctx *context.Context) { ctx.ServerError("DeleteUser", err) } } else { + go repo.StopJobsByUserID(ctx.User.ID) log.Trace("Account deleted: %s", ctx.User.Name) ctx.Redirect(setting.AppSubURL + "/") } diff --git a/templates/explore/dataset_list.tmpl b/templates/explore/dataset_list.tmpl index 48ae78127..7abc03363 100755 --- a/templates/explore/dataset_list.tmpl +++ b/templates/explore/dataset_list.tmpl @@ -29,8 +29,12 @@ {{.Repo.OwnerName}} / {{.Title}}
- {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}} - {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}} + {{if .Task}} + {{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}} + {{end}} + {{if .Category}} + {{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}} + {{end}} {{svg "octicon-flame" 16}} {{.DownloadTimes}}
diff --git a/templates/explore/repo_orgtop.tmpl b/templates/explore/repo_orgtop.tmpl index f0e6d9118..df132e811 100755 --- a/templates/explore/repo_orgtop.tmpl +++ b/templates/explore/repo_orgtop.tmpl @@ -52,6 +52,13 @@
+ + 京东 + +
+
+
+
TensorLayer diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 3af96998c..9099cb17a 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -215,7 +215,7 @@
-
+ + +
+ +
+
+ {{$.i18n.Tr "repo.modelarts.notebook"}} + +
+
+
+ +
+ {{svg "octicon-server" 16}} +
CPU / GPU
+ +
+ CPU / GPU + Ascend NPU +
+
+ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.modelarts.train_job.new_debug"}}{{end}} +
@@ -299,7 +323,7 @@ {{else}} {{.Status}} {{end}} --> - {{.Status}} + {{.Status}} - - 调试 + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.debug"}} + + {{else}} + + {{$.i18n.Tr "repo.debug"}} + {{end}} -
+ {{$.CsrfTokenHtml}} - - 停止 + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.stop"}} + + {{else}} + + {{$.i18n.Tr "repo.stop"}} + {{end}}
- 模型下载 + {{$.i18n.Tr "repo.download"}} - 提交镜像 - + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.submit_image"}} + {{else}} + {{$.i18n.Tr "repo.submit_image"}} + {{end}}
-
+ {{$.CsrfTokenHtml}} - - 删除 + {{if $.Permission.CanWrite $.UnitTypeCloudBrain}} + + {{$.i18n.Tr "repo.delete"}} + {{else}} + + {{$.i18n.Tr "repo.delete"}} + + {{end}}
@@ -488,24 +533,52 @@ $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { - console.log("---------",index,job) const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() == 'STOPPED') { + if (job.textContent.trim() == 'STOPPED' || job.textContent.trim() == 'FAILED') { return } - + $.get(`/api/v1/repos/${repoPath}/cloudbrain/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - console.log("status",status) if (status != job.textContent.trim()) { - //$('#' + jobID).text(status) + console.log("---------") + $('#' + jobID+'-icon').removeClass().addClass(status) + $('#' + jobID+ '-text').text(status) + //if (status == 'STOPPED') { - window.location.reload() + // window.location.reload() //} } + if(status==="RUNNING"){ + $('#model-debug-'+jobID).removeClass('disabled') + $('#model-debug-'+jobID).addClass('blue') + $('#model-image-'+jobID).removeClass('disabled') + $('#model-image-'+jobID).addClass('blue') + + + } + if(status!=="RUNNING"){ + $('#model-debug-'+jobID).removeClass('blue') + $('#model-debug-'+jobID).addClass('disabled') + $('#model-image-'+jobID).removeClass('blue') + $('#model-image-'+jobID).addClass('disabled') + + } + if(status!=="STOPPED" || status!=="FAILED"){ + $('#stop-model-debug-'+jobID).removeClass('disabled') + $('#stop-model-debug-'+jobID).addClass('blue') + $('#model-delete-'+jobID).removeClass('red') + $('#model-delete-'+jobID).addClass('disabled') + } + if(status=="STOPPED" || status=="FAILED"){ + $('#stop-model-debug-'+jobID).removeClass('blue') + $('#stop-model-debug-'+jobID).addClass('disabled') + $('#model-delete-'+jobID).removeClass('disabled') + $('#model-delete-'+jobID).addClass('red') + } }).fail(function(err) { console.log(err); }); diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index c45776b89..b3b827558 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -89,6 +89,10 @@ display: none; } + .inline.required.field.cloudbrain_brainscore { + display: none; + } + .select2-container .select2-selection--single{ height:38px !important; } @@ -116,7 +120,7 @@
{{template "base/alert" .}} -
+

@@ -130,12 +134,19 @@
-
+
@@ -209,11 +220,15 @@
- +
- + +
+
+ +
+{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/list_model.tmpl b/templates/repo/modelarts/trainjob/list_model.tmpl new file mode 100644 index 000000000..a441f5d99 --- /dev/null +++ b/templates/repo/modelarts/trainjob/list_model.tmpl @@ -0,0 +1,47 @@ +{{template "base/head" .}} +
+{{template "repo/header" .}} +
+
+ {{template "base/alert" .}} +

+
+
+ {{$.i18n.Tr "repo.modelarts.version_manage"}} +
+
+ {{svg "octicon-reply" 16}}{{$.i18n.Tr "repo.modelarts.back"}} +
+
+

+
+
+
+ + {{$.i18n.Tr "repo.modelarts.train_job.version"}} +
+
+
+ {{.ListModel}} +
+ +
+
semantic.json
+
Contains build settings for gulp
+
+
+ +
+
+
+ +
+
+ +
+
+{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/models/dir_list.tmpl b/templates/repo/modelarts/trainjob/models/dir_list.tmpl new file mode 100755 index 000000000..9e92681e5 --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/dir_list.tmpl @@ -0,0 +1,27 @@ +{{if .Dirs}} + + + {{range .Dirs}} + + + + + + {{end}} + +
+ + + + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} + + + + + {{.Size | FileSize}} + + + {{.ModTime}} +
+ +{{end}} diff --git a/templates/repo/modelarts/trainjob/models/index.tmpl b/templates/repo/modelarts/trainjob/models/index.tmpl new file mode 100755 index 000000000..9ca62dc2c --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/index.tmpl @@ -0,0 +1,29 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+
+

+ {{ range $index, $item := .Path }}{{ $item }}/{{ end }} +

+
+
+
+ +
+
+
+
+ {{template "repo/modelarts/trainjob/models/dir_list" .}} +
+
+
+
+
+
+ + + +{{template "base/footer" .}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl new file mode 100755 index 000000000..97b7b50e0 --- /dev/null +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -0,0 +1,606 @@ +{{template "base/head" .}} + + +
+
+
+
+
+
+
+
+
+
+ {{template "repo/header" .}} +
+ {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.modelarts.train_job.new"}} +

+
+ +
+ {{.CsrfTokenHtml}} + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

+
+ + +
+ +
+ + +
+ +
+ +

{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:

+ +
+ +
+ + +
+
+ + +
+ +
+ +
+ + {{if .bootFile}} + + {{else}} + + {{end}} + + + +
+
+ + +
+ +
+ + + + {{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} + +
+ +
+ + + + + + +
+ + +
+
+ + +
+ + + + +
+ +
+ + + +
+ + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+ + + +
+
+
+
+{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/para_manage.tmpl b/templates/repo/modelarts/trainjob/para_manage.tmpl new file mode 100755 index 000000000..b8ac6c78a --- /dev/null +++ b/templates/repo/modelarts/trainjob/para_manage.tmpl @@ -0,0 +1,154 @@ +{{template "base/head" .}} +
+
+ {{template "repo/header" .}} +
+
+ {{template "repo/modelarts/navbar" .}} + +
+
+
+

{{.i18n.Tr "repo.modelarts.train_job_para_admin"}}

+
+
+
+
+
+
+
+
+
+
+
+
+ + {{.i18n.Tr "repo.issues.filter_sort"}} + +
+
+
+
+ + +
+ {{range .Tasks}} +
+
+ +
+ + {{svg "octicon-tasklist" 16}} + {{.JobName}} + +
+ + +
+ {{.Status}} +
+ + +
+ {{svg "octicon-clock" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}} +
+ + +
+ + 编辑 + +
+ + +
+
+
+ {{$.CsrfTokenHtml}} + 删除 +
+
+
+ +
+
+ {{end}} {{template "base/paginate" .}} +
+
+
+
+
+
+
+
+ + +
+
+
+ 删除任务 +
+ +
+

你确认删除该任务么？此任务一旦删除不可恢复。

+
+
+
+ 取消操作 +
+
+ 确定操作 +
+
+
+
+
+{{template "base/footer" .}} + + diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl new file mode 100755 index 000000000..58a639ff5 --- /dev/null +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -0,0 +1,200 @@ +{{template "base/head" .}} +
+{{template "repo/header" .}} +
+
+ {{template "base/alert" .}} +

+
+
+ {{$.i18n.Tr "repo.modelarts.version_manage"}} +
+
+ {{svg "octicon-reply" 16}}{{$.i18n.Tr "repo.modelarts.back"}} +
+
+

+ +
+ +
+
+ + {{$.i18n.Tr "repo.modelarts.train_job.version"}} +
+
+
+
+ {{$.i18n.Tr "repo.modelarts.train_job.config"}} + {{$.i18n.Tr "repo.modelarts.log"}} + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}
{{.i18n.Tr "repo.modelarts.train_job.job_name"}} {{.result.JobName}}
{{.i18n.Tr "repo.modelarts.train_job.job_status"}} {{.result.Status}}
{{.i18n.Tr "repo.modelarts.train_job.version"}} {{.result.VersionName}}
{{.i18n.Tr "repo.modelarts.train_job.start_time"}} {{.result.CreateTime}}
{{.i18n.Tr "repo.modelarts.train_job.dura_time"}} {{.result.TrainJobDuration}}
{{.i18n.Tr "repo.modelarts.train_job.description"}} {{.result.Description}}
+
+
+ + + + + + + + + + + + + + + + + + + + + + +
{{.i18n.Tr "repo.modelarts.train_job.parameter_setting_info"}}
{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}} {{.result.EngineName}} | {{.result.EngineVersion}}
{{.i18n.Tr "repo.modelarts.train_job.start_file"}}{{.result.BootFileUrl}}
{{.i18n.Tr "repo.modelarts.train_job.dataset"}} {{.result.DatasetName}}
{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}} {{.result.Parameter}}
+
+
+ + + + + + + + + + + + + + + + + + +
{{.i18n.Tr "repo.modelarts.train_job.resource_setting_info"}}
{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}} {{.result.PoolName}}
{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}{{.result.WorkServerNum}}
{{.i18n.Tr "repo.modelarts.train_job.NAS_mount_path"}} {{.result.NasMountPath}}
+
+
+
+
+ +
+
+ + {{.log_file_name}} + + + +
+
+
+ 
{{.log.Content}}
+
+ +
+
+
+
+
+
+
+
+
+{{template "base/footer" .}} + + \ No newline at end of file diff --git a/vendor/modules.txt b/vendor/modules.txt index dbe4f72b4..d855d421d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -444,6 +444,7 @@ github.com/golang/protobuf/ptypes/timestamp # github.com/golang/snappy v0.0.1 github.com/golang/snappy # github.com/gomodule/redigo v2.0.0+incompatible +## explicit github.com/gomodule/redigo/internal github.com/gomodule/redigo/redis # github.com/google/go-github/v24 v24.0.1 diff --git a/web_src/js/components/EditTopics.vue b/web_src/js/components/EditTopics.vue index d2c871e03..d2109e178 100644 --- a/web_src/js/components/EditTopics.vue +++ b/web_src/js/components/EditTopics.vue @@ -11,16 +11,16 @@
-
{{arr.topic_name}}
+
{{arr.topic_name.toLowerCase()}}
- 点击或回车添加{{input}}标签 + 点击或回车添加{{input.toLowerCase()}}标签
-
{{input}}
+
{{input.toLowerCase()}}
@@ -134,7 +134,7 @@ export default { this.showSearchTopic = true } - else if(this.arrayTopics.indexOf(this.input)>-1){ + else if(this.arrayTopics.indexOf(this.input.toLowerCase())>-1){ this.showInputValue = false this.showSearchTopic = false @@ -142,7 +142,7 @@ export default { this.showInitTopic = [] let timestamp=new Date().getTime() - this.params.q = this.input + this.params.q = this.input.toLowerCase() this.params._ = timestamp this.$axios.get('/api/v1/topics/search',{ params:this.params @@ -164,7 +164,7 @@ export default { let findelement = this.array.some((item)=>{ - return item.topic_name===this.input + return item.topic_name===this.input.toLowerCase() }) this.showInputValue = !findelement @@ -224,11 +224,11 @@ export default { return }else{ let topic = this.input - if(this.arrayTopics.includes(topic)){ + if(this.arrayTopics.includes(topic.toLowerCase())){ return } else{ - this.arrayTopics.push(topic) + this.arrayTopics.push(topic.toLowerCase()) let topics = this.arrayTopics let strTopics = topics.join(',') @@ -250,7 +250,10 @@ export default { addPostTopic(){ if(this.showAddFlage){ - this.arrayTopics.pop() + // this.arrayTopics.pop() + + let cancleIndex = this.arrayTopics.indexOf(this.input) + this.arrayTopics.splice(cancleIndex,1) let topics = this.arrayTopics let strTopics = topics.join(',') let data = this.qs.stringify({ @@ -268,7 +271,7 @@ export default { } else if(!this.showAddFlage){ let topic = this.input - this.arrayTopics.push(topic) + this.arrayTopics.push(topic.toLowerCase()) let topics = this.arrayTopics let strTopics = topics.join(',') diff --git a/web_src/less/openi.less b/web_src/less/openi.less index cf8ca6d27..56234586e 100644 --- a/web_src/less/openi.less +++ b/web_src/less/openi.less @@ -224,15 +224,17 @@ footer .column{margin-bottom:0!important; padding-bottom:0!important;} // icon cloudbrain .i-round{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;} .i-bg-organ{background-position: -496px -52px;} -.STOPPED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -459px -52px;} +.STOPPED, .KILLED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -459px -52px;} .RUNNING{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -478px -52px;} .i-bg-orange{background-position: -495px -51px;} -.FAILED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -532px -52px;} +.FAILED,.START_FAILED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -532px -52px;} .i-bg-green{background-position: -441px -52px;} .i-bg-used{background-position: -514px -52px;} .icon-bind{background-position: -550px -52px;} .icon-unbind{background-position: -568px -52px;} -.CREATING, .STOPPING, .DELETING, .STARTING, .WAITING{display:inline-block;background-image:url('/img/loading.gif');background-repeat:no-repeat;width:16px;height:16px;background-size:16px 16px;margin-right:5px;} +.CREATING, .STOPPING, .DELETING, .STARTING, .WAITING ,.INIT,.KILLING{display:inline-block;background-image:url('/img/loading.gif');background-repeat:no-repeat;width:16px;height:16px;background-size:16px 16px;margin-right:5px;} + +.COMPLETED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -441px -52px;} .text_over{ overflow: hidden; text-overflow: ellipsis; diff --git a/web_src/less/themes/theme-arc-green.less b/web_src/less/themes/theme-arc-green.less old mode 100644 new mode 100755