diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go
index 5b14b9ba2..702cf0937 100644
--- a/models/ai_model_manage.go
+++ b/models/ai_model_manage.go
@@ -33,6 +33,7 @@ type AiModelManage struct {
CodeBranch string `xorm:"varchar(400) NULL" json:"codeBranch"`
CodeCommitID string `xorm:"NULL" json:"codeCommitID"`
UserId int64 `xorm:"NOT NULL" json:"userId"`
+ IsPrivate bool `xorm:"DEFAULT true" json:"isPrivate"`
UserName string `json:"userName"`
UserRelAvatarLink string `json:"userRelAvatarLink"`
TrainTaskInfo string `xorm:"text NULL" json:"trainTaskInfo"`
@@ -40,6 +41,7 @@ type AiModelManage struct {
UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"`
IsCanOper bool `json:"isCanOper"`
IsCanDelete bool `json:"isCanDelete"`
+ IsCanDownload bool `json:"isCanDownload"`
}
type AiModelConvert struct {
@@ -84,8 +86,10 @@ type AiModelQueryOptions struct {
SortType string
New int
// JobStatus CloudbrainStatus
- Type int
- Status int
+ Type int
+ Status int
+ IsOnlyThisRepo bool
+ IsQueryPrivate bool
}
func (a *AiModelConvert) IsGpuTrainTask() bool {
@@ -217,6 +221,19 @@ func SaveModelToDb(model *AiModelManage) error {
return nil
}
+func QueryModelConvertByName(name string, repoId int64) ([]*AiModelConvert, error) {
+ sess := x.NewSession()
+ defer sess.Close()
+ sess.Select("*").Table(new(AiModelConvert)).
+ Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("created_unix desc")
+ aiModelManageConvertList := make([]*AiModelConvert, 0)
+ err := sess.Find(&aiModelManageConvertList)
+ if err == nil {
+ return aiModelManageConvertList, nil
+ }
+ return nil, err
+}
+
func QueryModelConvertById(id string) (*AiModelConvert, error) {
sess := x.NewSession()
defer sess.Close()
@@ -288,15 +305,30 @@ func ModifyModelDescription(id string, description string) error {
return nil
}
-func ModifyLocalModel(id string, name, label, description string, engine int) error {
+func ModifyModelPrivate(id string, isPrivate bool) error {
+ var sess *xorm.Session
+ sess = x.ID(id)
+ defer sess.Close()
+ re, err := sess.Cols("is_private").Update(&AiModelManage{
+ IsPrivate: isPrivate,
+ })
+ if err != nil {
+ return err
+ }
+ log.Info("success to update isPrivate from db.re=" + fmt.Sprint((re)))
+ return nil
+}
+
+func ModifyLocalModel(id string, name, label, description string, engine int, isPrivate bool) error {
var sess *xorm.Session
sess = x.ID(id)
defer sess.Close()
- re, err := sess.Cols("name", "label", "description", "engine").Update(&AiModelManage{
+ re, err := sess.Cols("name", "label", "description", "engine", "is_private").Update(&AiModelManage{
Description: description,
Name: name,
Label: label,
Engine: int64(engine),
+ IsPrivate: isPrivate,
})
if err != nil {
return err
@@ -411,7 +443,11 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) {
builder.Eq{"ai_model_manage.status": opts.Status},
)
}
-
+ if !opts.IsQueryPrivate {
+ cond = cond.And(
+ builder.Eq{"ai_model_manage.is_private": false},
+ )
+ }
count, err := sess.Where(cond).Count(new(AiModelManage))
if err != nil {
return nil, 0, fmt.Errorf("Count: %v", err)
diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 1e7c702ab..aeed8629c 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -204,6 +204,7 @@ type Cloudbrain struct {
BenchmarkTypeRankLink string `xorm:"-"`
StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp
+ Cleared bool `xorm:"DEFAULT false"`
Spec *Specification `xorm:"-"`
}
@@ -1905,6 +1906,12 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) {
return getRepoCloudBrain(cb)
}
+func IsCloudbrainExistByJobName(jobName string)(bool,error){
+ return x.Unscoped().Exist(&Cloudbrain{
+ JobName: jobName,
+ })
+}
+
func GetCloudbrainByIDWithDeleted(id string) (*Cloudbrain, error) {
idInt64, _ := strconv.ParseInt(id, 10, 64)
cb := &Cloudbrain{ID: idInt64}
@@ -2050,6 +2057,83 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) {
Find(&cloudbrains)
}
+func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
+ cloudbrains := make([]*Cloudbrain, 0, 10)
+ endTimeBefore := time.Now().Unix() - int64(days)*24*3600
+ missEndTimeBefore := endTimeBefore - 24*3600
+ return cloudbrains, x.Unscoped().Cols("id,job_name,job_id").
+ In("status",
+ JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted,
+ ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed,
+ ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed).
+ Where("(((end_time is null or end_time=0) and updated_unix 0 {
+ idsIn := ""
+ for i, id := range tempIds {
+ if i == 0 {
+ idsIn += strconv.FormatInt(id, 10)
+ } else {
+ idsIn += "," + strconv.FormatInt(id, 10)
+ }
+ }
+
+ _, errTemp := x.Unscoped().Exec("update cloudbrain set cleared=true where id in (" + idsIn + ")")
+ if errTemp != nil {
+ err = errTemp
+ }
+
+ }
+
+ }
+ return err
+
+}
+
+func getPageIds(ids []int64, page int, pagesize int) []int64 {
+ begin := (page - 1) * pagesize
+ end := (page) * pagesize
+
+ if begin > len(ids)-1 {
+ return []int64{}
+ }
+ if end > len(ids)-1 {
+ return ids[begin:]
+ } else {
+ return ids[begin:end]
+ }
+
+}
+
func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0)
return cloudbrains, x.
diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index a213f179c..40d7a2a2e 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -183,6 +183,17 @@ func GetWaittingTop() ([]*CloudbrainInfo, error) {
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
+
+ var ids []int64
+ for _, task := range cloudbrains {
+ ids = append(ids, task.RepoID)
+ }
+ repositoryMap, err := GetRepositoriesMapByIDs(ids)
+ if err == nil {
+ for _, task := range cloudbrains {
+ task.Repo = repositoryMap[task.RepoID]
+ }
+ }
return cloudbrains, nil
}
@@ -199,6 +210,16 @@ func GetRunningTop() ([]*CloudbrainInfo, error) {
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
+ var ids []int64
+ for _, task := range cloudbrains {
+ ids = append(ids, task.RepoID)
+ }
+ repositoryMap, err := GetRepositoriesMapByIDs(ids)
+ if err == nil {
+ for _, task := range cloudbrains {
+ task.Repo = repositoryMap[task.RepoID]
+ }
+ }
return cloudbrains, nil
}
diff --git a/models/resource_specification.go b/models/resource_specification.go
index 2f815818b..809a3496a 100644
--- a/models/resource_specification.go
+++ b/models/resource_specification.go
@@ -3,6 +3,7 @@ package models
import (
"code.gitea.io/gitea/modules/timeutil"
"fmt"
+ "strings"
"xorm.io/builder"
)
@@ -197,12 +198,104 @@ type Specification struct {
AiCenterName string
IsExclusive bool
ExclusiveOrg string
+ //specs that have the same sourceSpecId, computeResource and cluster as current spec
+ RelatedSpecs []*Specification
}
func (Specification) TableName() string {
return "resource_specification"
}
+func (s *Specification) loadRelatedSpecs() {
+ if s.RelatedSpecs != nil {
+ return
+ }
+ defaultSpecs := make([]*Specification, 0)
+ if s.SourceSpecId == "" {
+ s.RelatedSpecs = defaultSpecs
+ return
+ }
+ r, err := FindSpecs(FindSpecsOptions{
+ ComputeResource: s.ComputeResource,
+ Cluster: s.Cluster,
+ SourceSpecId: s.SourceSpecId,
+ RequestAll: true,
+ SpecStatus: SpecOnShelf,
+ })
+ if err != nil {
+ s.RelatedSpecs = defaultSpecs
+ return
+ }
+ s.RelatedSpecs = r
+}
+func (s *Specification) GetAvailableCenterIds(userIds ...int64) []string {
+ s.loadRelatedSpecs()
+
+ if len(s.RelatedSpecs) == 0 {
+ return make([]string, 0)
+ }
+
+ var uId int64
+ if len(userIds) > 0 {
+ uId = userIds[0]
+ }
+ //filter exclusive specs
+ specs := FilterExclusiveSpecs(s.RelatedSpecs, uId)
+
+ centerIds := make([]string, len(specs))
+ for i, v := range specs {
+ centerIds[i] = v.AiCenterCode
+ }
+ return centerIds
+}
+
+func FilterExclusiveSpecs(r []*Specification, userId int64) []*Specification {
+ if userId == 0 {
+ return r
+ }
+ specs := make([]*Specification, 0, len(r))
+ specMap := make(map[int64]string, 0)
+ for i := 0; i < len(r); i++ {
+ spec := r[i]
+ if _, has := specMap[spec.ID]; has {
+ continue
+ }
+ if !spec.IsExclusive {
+ specs = append(specs, spec)
+ specMap[spec.ID] = ""
+ continue
+ }
+ orgs := strings.Split(spec.ExclusiveOrg, ";")
+ for _, org := range orgs {
+ isMember, _ := IsOrganizationMemberByOrgName(org, userId)
+ if isMember {
+ specs = append(specs, spec)
+ specMap[spec.ID] = ""
+ break
+ }
+ }
+ }
+ return specs
+}
+
+func DistinctSpecs(r []*Specification) []*Specification {
+ specs := make([]*Specification, 0, len(r))
+ sourceSpecIdMap := make(map[string]string, 0)
+ for i := 0; i < len(r); i++ {
+ spec := r[i]
+ if spec.SourceSpecId == "" {
+ specs = append(specs, spec)
+ continue
+ }
+ if _, has := sourceSpecIdMap[spec.SourceSpecId]; has {
+ continue
+ }
+ specs = append(specs, spec)
+ sourceSpecIdMap[spec.SourceSpecId] = ""
+ }
+ return specs
+}
+
func InsertResourceSpecification(r ResourceSpecification) (int64, error) {
return x.Insert(&r)
}
diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go
index 2066697d2..99ff990ce 100644
--- a/models/user_analysis_for_activity.go
+++ b/models/user_analysis_for_activity.go
@@ -449,3 +449,20 @@ func QueryUserLoginInfo(userIds []int64) []*UserLoginLog {
return loginList
}
+
+func QueryUserAnnualReport(userId int64) *UserSummaryCurrentYear {
+ statictisSess := xStatistic.NewSession()
+ defer statictisSess.Close()
+ log.Info("userId=" + fmt.Sprint(userId))
+
+ reList := make([]*UserSummaryCurrentYear, 0)
+ err := statictisSess.Select("*").Table(new(UserSummaryCurrentYear)).Where("id=" + fmt.Sprint(userId)).Find(&reList)
+ if err == nil {
+ if len(reList) > 0 {
+ return reList[0]
+ }
+ } else {
+ log.Info("error:=" + err.Error())
+ }
+ return nil
+}
diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go
index 394c24825..6c247709e 100644
--- a/models/user_business_analysis.go
+++ b/models/user_business_analysis.go
@@ -132,11 +132,17 @@ func makeResultForMonth(allUserInfo []*UserMetrics, count int) []*UserMetrics {
if count > 0 {
for _, userMetrics := range allUserInfo {
dateTime := time.Unix(userMetrics.CountDate, 0)
- month := fmt.Sprint(dateTime.Year()) + "-" + fmt.Sprint(int(dateTime.Month()))
+ mInt := int(dateTime.Month())
+ mString := fmt.Sprint(mInt)
+ if mInt < 10 {
+ mString = "0" + mString
+ }
+ month := fmt.Sprint(dateTime.Year()) + "-" + mString
if _, ok := monthMap[month]; !ok {
monthUserMetrics := &UserMetrics{
DisplayDate: month,
ActivateRegistUser: userMetrics.ActivateRegistUser,
+ RegistActivityUser: userMetrics.RegistActivityUser,
NotActivateRegistUser: userMetrics.NotActivateRegistUser,
TotalUser: userMetrics.TotalUser,
TotalNotActivateRegistUser: userMetrics.TotalUser - userMetrics.TotalActivateRegistUser,
@@ -152,6 +158,7 @@ func makeResultForMonth(allUserInfo []*UserMetrics, count int) []*UserMetrics {
value.ActivateRegistUser += userMetrics.ActivateRegistUser
value.NotActivateRegistUser += userMetrics.NotActivateRegistUser
value.HasActivityUser += userMetrics.HasActivityUser
+ value.RegistActivityUser += userMetrics.RegistActivityUser
value.TotalRegistUser += userMetrics.ActivateRegistUser + userMetrics.NotActivateRegistUser
value.ActivateIndex = float64(value.ActivateRegistUser) / float64(value.TotalRegistUser)
value.DaysForMonth += 1
@@ -610,7 +617,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
DataDate := currentTimeNow.Format("2006-01-02") + " 00:01"
bonusMap := make(map[string]map[string]int)
- if tableName == "user_business_analysis_current_year" {
+ if isUserYearData(tableName) {
bonusMap = getBonusMap()
log.Info("truncate all data from table:user_summary_current_year ")
statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")
@@ -712,7 +719,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1
}
}
- if tableName == "user_business_analysis_current_year" {
+ if isUserYearData(tableName) {
//年度数据
subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
mostActiveDay := ""
@@ -772,6 +779,17 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount))
}
+func isUserYearData(tableName string) bool {
+ if tableName == "user_business_analysis_current_year" {
+ currentTimeNow := time.Now()
+ if currentTimeNow.Year() >= 2023 {
+ return false
+ }
+ return true
+ }
+ return false
+}
+
func getBonusMap() map[string]map[string]int {
bonusMap := make(map[string]map[string]int)
url := setting.RecommentRepoAddr + "bonus/record.txt"
@@ -794,6 +812,7 @@ func getBonusMap() map[string]map[string]int {
record, ok := bonusMap[userName]
if !ok {
record = make(map[string]int)
+ bonusMap[userName] = record
}
record["times"] = getMapKeyStringValue("times", record) + getIntValue(aLine[3])
record["total_bonus"] = getMapKeyStringValue("total_bonus", record) + getIntValue(aLine[4])
diff --git a/models/user_invitation.go b/models/user_invitation.go
index 2d37bcb23..8a10e71a0 100644
--- a/models/user_invitation.go
+++ b/models/user_invitation.go
@@ -13,6 +13,7 @@ type Invitation struct {
SrcUserID int64 `xorm:"NOT NULL DEFAULT 0"`
UserID int64 `xorm:"NOT NULL DEFAULT 0"`
Phone string `xorm:"INDEX"`
+ Email string `xorm:"-"`
Avatar string `xorm:"-"`
Name string `xorm:"-"`
InvitationUserNum int `xorm:"-"`
diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go
index 985a82cdb..6a1fc6e39 100755
--- a/modules/cron/tasks_basic.go
+++ b/modules/cron/tasks_basic.go
@@ -5,10 +5,13 @@
package cron
import (
- "code.gitea.io/gitea/modules/urfs_client/urchin"
+ "code.gitea.io/gitea/modules/setting"
"context"
"time"
+ "code.gitea.io/gitea/modules/urfs_client/urchin"
+ cloudbrainService "code.gitea.io/gitea/services/cloudbrain"
+
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/services/cloudbrain/resource"
"code.gitea.io/gitea/services/reward"
@@ -190,6 +193,17 @@ func registerHandleRepoAndUserStatistic() {
})
}
+func registerHandleClearCloudbrainResult() {
+ RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{
+ Enabled: true,
+ RunAtStart: setting.ClearStrategy.RunAtStart,
+ Schedule: setting.ClearStrategy.Cron,
+ }, func(ctx context.Context, _ *models.User, _ Config) error {
+ cloudbrainService.ClearCloudbrainResultSpace()
+ return nil
+ })
+}
+
func registerHandleSummaryStatistic() {
RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
Enabled: true,
@@ -306,6 +320,7 @@ func initBasicTasks() {
registerHandleRepoAndUserStatistic()
registerHandleSummaryStatistic()
+ registerHandleClearCloudbrainResult()
registerSyncCloudbrainStatus()
registerHandleOrgStatistic()
@@ -317,6 +332,6 @@ func initBasicTasks() {
registerHandleModelSafetyTask()
- registerHandleScheduleRecord()
+ registerHandleScheduleRecord()
registerHandleCloudbrainDurationStatistic()
}
diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go
index 280407240..34d7d3fe0 100755
--- a/modules/grampus/grampus.go
+++ b/modules/grampus/grampus.go
@@ -105,8 +105,6 @@ func getDatasetGrampus(datasetInfos map[string]models.DatasetInfo) []models.Gram
func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) {
createTime := timeutil.TimeStampNow()
- centerID, centerName := getCentersParamter(ctx, req)
-
var datasetGrampus, modelGrampus []models.GrampusDataset
var codeGrampus models.GrampusDataset
if ProcessorTypeNPU == req.ProcessType {
@@ -138,8 +136,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
ResourceSpecId: req.Spec.SourceSpecId,
ImageId: req.ImageId,
ImageUrl: req.ImageUrl,
- CenterID: centerID,
- CenterName: centerName,
+ CenterID: req.Spec.GetAvailableCenterIds(ctx.User.ID),
ReplicaNum: 1,
Datasets: datasetGrampus,
Models: modelGrampus,
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index a473dad6a..bf7eb2352 100755
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -519,6 +519,7 @@ var (
CullIdleTimeout string
CullInterval string
+
//benchmark config
IsBenchmarkEnabled bool
BenchmarkOwner string
@@ -613,6 +614,16 @@ var (
UsageRateBeginTime string
}{}
+ ClearStrategy= struct {
+ Enabled bool
+ ResultSaveDays int
+ BatchSize int
+ DebugJobSize int
+ TrashSaveDays int
+ Cron string
+ RunAtStart bool
+ }{}
+
C2NetInfos *C2NetSqInfos
CenterInfos *AiCenterInfos
C2NetMapInfo map[string]*C2NetSequenceInfo
@@ -1619,6 +1630,7 @@ func NewContext() {
getModelConvertConfig()
getModelSafetyConfig()
getModelAppConfig()
+ getClearStrategy()
}
func getModelSafetyConfig() {
@@ -1679,6 +1691,18 @@ func getModelartsCDConfig() {
getNotebookFlavorInfos()
}
+func getClearStrategy(){
+
+ sec := Cfg.Section("clear_strategy")
+ ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false)
+ ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30)
+ ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500)
+ ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100)
+ ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90)
+ ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?")
+ ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false)
+}
+
func getGrampusConfig() {
sec := Cfg.Section("grampus")
diff --git a/modules/structs/pipeline.go b/modules/structs/pipeline.go
new file mode 100644
index 000000000..fd26d1b51
--- /dev/null
+++ b/modules/structs/pipeline.go
@@ -0,0 +1,23 @@
+package structs
+
+type Pipeline struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Status string `json:"status"`
+}
+type NodeInfo struct {
+ Name string `json:"name"`
+ Status string `json:"status"`
+ Code string `json:"code"`
+ Message string `json:"message"`
+}
+
+type PipelineNotification struct {
+ Type int `json:"type"`
+ Username string `json:"username"`
+ Reponame string `json:"reponame"`
+ Pipeline Pipeline `json:"pipeline"`
+ PipelineRunId string `json:"pipeline_run_id"`
+ Node NodeInfo `json:"node"`
+ OccurTime int64 `json:"occur_time"`
+}
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 647bdb1ad..082b35ca8 100755
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -1305,6 +1305,11 @@ model.manage.select.engine=Select model engine
model.manage.modelfile=Model file
model.manage.modellabel=Model label
model.manage.modeldesc=Model description
+model.manage.modelaccess=Model Access
+model.manage.modelaccess.public=Public
+model.manage.modelaccess.private=Private
+model.manage.modelaccess.setpublic=Set Public
+model.manage.modelaccess.setprivate=Set Private
model.manage.baseinfo=Base Information
modelconvert.notcreate=No model conversion task has been created.
modelconvert.importfirst1=Please import the
@@ -3246,6 +3251,7 @@ specification = specification
select_specification = select specification
description = description
wrong_specification=You cannot use this specification, please choose another item.
+result_cleared=The files of the task have been cleared, can not restart any more, please create a new debug task instead.
resource_use=Resource Occupancy
job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-).
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index 8f9e6b664..f4e8f1aea 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -1318,6 +1318,11 @@ model.manage.select.engine=选择模型框架
model.manage.modelfile=模型文件
model.manage.modellabel=模型标签
model.manage.modeldesc=模型描述
+model.manage.modelaccess=模型权限
+model.manage.modelaccess.public=公开
+model.manage.modelaccess.private=私有
+model.manage.modelaccess.setpublic=设为公开
+model.manage.modelaccess.setprivate=设为私有
model.manage.baseinfo=基本信息
modelconvert.notcreate=未创建过模型转换任务
modelconvert.importfirst1=请您先导入
@@ -3266,6 +3271,8 @@ card_duration = 运行卡时
card_type = 卡类型
wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。
+result_cleared=本任务的文件已被清理，无法再次调试，请新建调试任务。
+
job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。
train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。
infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index 99574c620..117484689 100755
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -544,6 +544,10 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/complete_multipart", repo.CompleteMultipart)
}, reqToken())
+ m.Group("/pipeline", func() {
+ m.Post("/notification", bind(api.PipelineNotification{}), notify.PipelineNotify)
+
+ }, reqToken())
// Notifications
m.Group("/notifications", func() {
@@ -610,6 +614,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/query_invitation_yesterday", operationReq, repo_ext.QueryInvitationYesterday)
m.Get("/query_invitation_all", operationReq, repo_ext.QueryInvitationAll)
m.Get("/query_invitation_userdefine", operationReq, repo_ext.QueryUserDefineInvitationPage)
+ m.Get("/query_user_annual_report", repo_ext.QueryUserAnnualReport)
m.Get("/download_invitation_detail", operationReq, repo_ext.DownloadInvitationDetail)
@@ -758,6 +763,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:username/:reponame", func() {
m.Get("/right", reqToken(), repo.GetRight)
m.Get("/tagger", reqToken(), repo.ListTagger)
+ m.Get("/cloudBrainJobId", repo.GetCloudBrainJobId)
m.Combo("").Get(reqAnyRepoReader(), repo.Get).
Delete(reqToken(), reqOwner(), repo.Delete).
Patch(reqToken(), reqAdmin(), bind(api.EditRepoOption{}), context.RepoRef(), repo.Edit)
@@ -998,6 +1004,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/detail", reqToken(), reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow)
m.Get("/model_list", repo.CloudBrainModelList)
m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.CloudBrainStop)
+ m.Put("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GeneralCloudBrainJobStop)
})
})
m.Group("/inference-job", func() {
@@ -1018,12 +1025,15 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Delete("/delete_model", repo.DeleteModel)
m.Get("/downloadall", repo.DownloadModel)
m.Get("/query_model_byId", repo.QueryModelById)
+ m.Get("/query_model_byName", repo.QueryModelByName)
m.Get("/query_model_for_predict", repo.QueryModelListForPredict)
m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict)
m.Get("/query_train_model", repo.QueryTrainModelList)
m.Post("/create_model_convert", repo.CreateModelConvert)
+ m.Post("/convert_stop", repo.StopModelConvert)
m.Get("/show_model_convert_page", repo.ShowModelConvertPage)
m.Get("/query_model_convert_byId", repo.QueryModelConvertById)
+ m.Get("/query_model_convert_byName", repo.QueryModelConvertByName)
m.Get("/:id", repo.GetCloudbrainModelConvertTask)
m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog)
diff --git a/routers/api/v1/notify/pipeline.go b/routers/api/v1/notify/pipeline.go
new file mode 100644
index 000000000..021af20dc
--- /dev/null
+++ b/routers/api/v1/notify/pipeline.go
@@ -0,0 +1,15 @@
+package notify
+
+import (
+ "net/http"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/context"
+ api "code.gitea.io/gitea/modules/structs"
+)
+
+func PipelineNotify(ctx *context.APIContext, form api.PipelineNotification) {
+
+ ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
+
+}
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index cd8340c41..1c5a58b47 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -17,6 +17,8 @@ import (
"strings"
"time"
+ "code.gitea.io/gitea/modules/grampus"
+
cloudbrainService "code.gitea.io/gitea/services/cloudbrain"
"code.gitea.io/gitea/modules/convert"
@@ -80,6 +82,30 @@ func CloudBrainShow(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)})
}
+func GeneralCloudBrainJobStop(ctx *context.APIContext) {
+ task := ctx.Cloudbrain
+ if task.IsTerminal() {
+ ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("cloudbrain.Already_stopped"))
+ return
+ }
+ var err error
+
+ if ctx.Cloudbrain.Type == models.TypeCloudBrainOne {
+ err = cloudbrain.StopJob(task.JobID)
+ } else if ctx.Cloudbrain.Type == models.TypeCloudBrainTwo {
+ _, err = modelarts.StopTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
+ } else {
+ _, err = grampus.StopJob(task.JobID)
+ }
+
+ if err != nil {
+ log.Warn("cloud brain stopped failed.", err)
+ ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("cloudbrain.Stopped_failed"))
+ return
+ }
+
+ ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
+}
func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) {
cloudbrainTask.FileNotebookCreate(ctx.Context, option)
}
diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index 7fe5d603c..0d68fff30 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -968,6 +968,8 @@ func GetWaittingTop(ctx *context.Context) {
taskDetail.RepoID = ciTasks[i].RepoID
if ciTasks[i].Repo != nil {
taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name
+ } else {
+ taskDetail.RepoName = ""
}
WaitTimeInt := time.Now().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix()
taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt)
@@ -975,6 +977,13 @@ func GetWaittingTop(ctx *context.Context) {
if WaitTimeInt < 0 {
taskDetail.WaitTime = "00:00:00"
}
+
+ taskDetail.ID = ciTasks[i].Cloudbrain.ID
+ taskDetail.ComputeResource = ciTasks[i].Cloudbrain.ComputeResource
+ taskDetail.JobType = ciTasks[i].Cloudbrain.JobType
+ taskDetail.JobID = ciTasks[i].Cloudbrain.JobID
+ taskDetail.Type = ciTasks[i].Cloudbrain.Type
+
tasks = append(tasks, taskDetail)
}
ctx.JSON(http.StatusOK, map[string]interface{}{
@@ -1001,6 +1010,12 @@ func GetRunningTop(ctx *context.Context) {
taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name
}
+ taskDetail.ID = ciTasks[i].Cloudbrain.ID
+ taskDetail.ComputeResource = ciTasks[i].Cloudbrain.ComputeResource
+ taskDetail.JobType = ciTasks[i].Cloudbrain.JobType
+ taskDetail.JobID = ciTasks[i].Cloudbrain.JobID
+ taskDetail.Type = ciTasks[i].Cloudbrain.Type
+
tasks = append(tasks, taskDetail)
}
ctx.JSON(http.StatusOK, map[string]interface{}{
diff --git a/routers/api/v1/repo/mlops.go b/routers/api/v1/repo/mlops.go
index 43969330d..322edc3e5 100644
--- a/routers/api/v1/repo/mlops.go
+++ b/routers/api/v1/repo/mlops.go
@@ -69,3 +69,17 @@ func GetRight(ctx *context.APIContext) {
})
}
+
+func GetCloudBrainJobId(ctx *context.APIContext) {
+ cloudbrains, err := models.GetCloudbrainsByDisplayJobName(ctx.Repo.Repository.ID, ctx.Query("jobType"), ctx.Query("name"))
+ if err != nil {
+ log.Warn("get cloudbrain by display name failed", err)
+ ctx.JSON(http.StatusOK, map[string]string{"jobId": ""})
+ return
+ }
+ if len(cloudbrains) > 0 {
+ ctx.JSON(http.StatusOK, map[string]string{"jobId": cloudbrains[0].JobID})
+ return
+ }
+ ctx.JSON(http.StatusOK, map[string]string{"jobId": ""})
+}
diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go
index 15260790d..3b0aed0d5 100644
--- a/routers/api/v1/repo/modelmanage.go
+++ b/routers/api/v1/repo/modelmanage.go
@@ -43,8 +43,14 @@ func QueryModelById(ctx *context.APIContext) {
routerRepo.QueryModelById(ctx.Context)
}
+func QueryModelByName(ctx *context.APIContext) {
+ log.Info("QueryModelByName by api.")
+ routerRepo.ShowSingleModel(ctx.Context)
+}
+
func QueryModelListForPredict(ctx *context.APIContext) {
log.Info("QueryModelListForPredict by api.")
+ ctx.Context.SetParams("isOnlyThisRepo", "true")
routerRepo.QueryModelListForPredict(ctx.Context)
}
@@ -88,6 +94,11 @@ func CreateModelConvert(ctx *context.APIContext) {
routerRepo.SaveModelConvert(ctx.Context)
}
+func StopModelConvert(ctx *context.APIContext) {
+ log.Info("StopModelConvert by api.")
+ routerRepo.StopModelConvertApi(ctx.Context)
+}
+
func ShowModelConvertPage(ctx *context.APIContext) {
log.Info("ShowModelConvertPage by api.")
modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context)
@@ -113,3 +124,12 @@ func QueryModelConvertById(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, nil)
}
}
+
+func QueryModelConvertByName(ctx *context.APIContext) {
+ modelResult, err := routerRepo.GetModelConvertByName(ctx.Context)
+ if err == nil {
+ ctx.JSON(http.StatusOK, modelResult)
+ } else {
+ ctx.JSON(http.StatusOK, nil)
+ }
+}
diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go
index 4ba414bff..9839a5041 100644
--- a/routers/repo/ai_model_convert.go
+++ b/routers/repo/ai_model_convert.go
@@ -573,13 +573,10 @@ func deleteCloudBrainTask(task *models.AiModelConvert) {
}
}
-func StopModelConvert(ctx *context.Context) {
- id := ctx.Params(":id")
- log.Info("stop model convert start.id=" + id)
+func stopModelConvert(id string) error {
job, err := models.QueryModelConvertById(id)
if err != nil {
- ctx.ServerError("Not found task.", err)
- return
+ return err
}
if job.IsGpuTrainTask() {
err = cloudbrain.StopJob(job.CloudBrainTaskId)
@@ -600,6 +597,35 @@ func StopModelConvert(ctx *context.Context) {
err = models.UpdateModelConvert(job)
if err != nil {
log.Error("UpdateModelConvert failed:", err)
+ return err
+ }
+ return nil
+}
+
+func StopModelConvertApi(ctx *context.Context) {
+ id := ctx.Params(":id")
+ log.Info("stop model convert start.id=" + id)
+ err := stopModelConvert(id)
+ if err == nil {
+ ctx.JSON(200, map[string]string{
+ "code": "0",
+ "msg": "succeed",
+ })
+ } else {
+ ctx.JSON(200, map[string]string{
+ "code": "1",
+ "msg": err.Error(),
+ })
+ }
+}
+
+func StopModelConvert(ctx *context.Context) {
+ id := ctx.Params(":id")
+ log.Info("stop model convert start.id=" + id)
+ err := stopModelConvert(id)
+ if err != nil {
+ ctx.ServerError("Not found task.", err)
+ return
}
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelmanage/convert_model")
}
@@ -620,7 +646,7 @@ func ShowModelConvertInfo(ctx *context.Context) {
return
}
ctx.Data["Name"] = job.Name
- ctx.Data["canDownload"] = isOper(ctx, job.UserId)
+ ctx.Data["canDownload"] = isOperModifyOrDelete(ctx, job.UserId)
user, err := models.GetUserByID(job.UserId)
if err == nil {
job.UserName = user.Name
@@ -732,6 +758,11 @@ func GetModelConvertById(ctx *context.Context) (*models.AiModelConvert, error) {
return models.QueryModelConvertById(id)
}
+func GetModelConvertByName(ctx *context.Context) ([]*models.AiModelConvert, error) {
+ name := ctx.Query("name")
+ return models.QueryModelConvertByName(name, ctx.Repo.Repository.ID)
+}
+
func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) {
page := ctx.QueryInt("page")
if page <= 0 {
@@ -755,7 +786,7 @@ func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, in
}
userIds := make([]int64, len(modelResult))
for i, model := range modelResult {
- model.IsCanOper = isOper(ctx, model.UserId)
+ model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
model.IsCanDelete = isCanDelete(ctx, model.UserId)
userIds[i] = model.UserId
}
diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go
index 7eedb9bc4..6e6889c32 100644
--- a/routers/repo/ai_model_manage.go
+++ b/routers/repo/ai_model_manage.go
@@ -93,7 +93,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
log.Info("accuracyJson=" + string(accuracyJson))
aiTask.ContainerIp = ""
aiTaskJson, _ := json.Marshal(aiTask)
-
+ isPrivate := ctx.QueryBool("isPrivate")
model := &models.AiModelManage{
ID: id,
Version: version,
@@ -114,6 +114,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
TrainTaskInfo: string(aiTaskJson),
Accuracy: string(accuracyJson),
Status: STATUS_COPY_MODEL,
+ IsPrivate: isPrivate,
}
err = models.SaveModelToDb(model)
@@ -216,6 +217,7 @@ func SaveLocalModel(ctx *context.Context) {
description := ctx.Query("description")
engine := ctx.QueryInt("engine")
taskType := ctx.QueryInt("type")
+ isPrivate := ctx.QueryBool("isPrivate")
modelActualPath := ""
if taskType == models.TypeCloudBrainOne {
destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(id) + "/"
@@ -262,6 +264,7 @@ func SaveLocalModel(ctx *context.Context) {
TrainTaskInfo: "",
Accuracy: "",
Status: STATUS_FINISHED,
+ IsPrivate: isPrivate,
}
err := models.SaveModelToDb(model)
@@ -554,20 +557,6 @@ func deleteModelByID(ctx *context.Context, id string) error {
return err
}
-func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) {
-
- return models.QueryModel(&models.AiModelQueryOptions{
- ListOptions: models.ListOptions{
- Page: page,
- PageSize: setting.UI.IssuePagingNum,
- },
- RepoID: repoId,
- Type: -1,
- New: MODEL_LATEST,
- Status: -1,
- })
-}
-
func DownloadMultiModelFile(ctx *context.Context) {
log.Info("DownloadMultiModelFile start.")
id := ctx.Query("id")
@@ -578,7 +567,7 @@ func DownloadMultiModelFile(ctx *context.Context) {
ctx.ServerError("no such model:", err)
return
}
- if !isOper(ctx, task.UserId) {
+ if !isCanDownload(ctx, task) {
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
return
}
@@ -806,7 +795,7 @@ func DownloadSingleModelFile(ctx *context.Context) {
ctx.ServerError("no such model:", err)
return
}
- if !isOper(ctx, task.UserId) {
+ if !isCanDownload(ctx, task) {
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
return
}
@@ -874,8 +863,9 @@ func QueryModelById(ctx *context.Context) {
id := ctx.Query("id")
model, err := models.QueryModelById(id)
if err == nil {
- model.IsCanOper = isOper(ctx, model.UserId)
+ model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
model.IsCanDelete = isCanDelete(ctx, model.UserId)
+ model.IsCanDownload = isCanDownload(ctx, model)
removeIpInfo(model)
ctx.JSON(http.StatusOK, model)
} else {
@@ -891,7 +881,8 @@ func ShowSingleModel(ctx *context.Context) {
userIds := make([]int64, len(models))
for i, model := range models {
- model.IsCanOper = isOper(ctx, model.UserId)
+ model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
+ model.IsCanDownload = isCanDownload(ctx, model)
model.IsCanDelete = isCanDelete(ctx, model.UserId)
userIds[i] = model.UserId
}
@@ -941,7 +932,8 @@ func ShowOneVersionOtherModel(ctx *context.Context) {
userIds := make([]int64, len(aimodels))
for i, model := range aimodels {
- model.IsCanOper = isOper(ctx, model.UserId)
+ model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
+ model.IsCanDownload = isCanDownload(ctx, model)
model.IsCanDelete = isCanDelete(ctx, model.UserId)
userIds[i] = model.UserId
}
@@ -964,6 +956,7 @@ func ShowOneVersionOtherModel(ctx *context.Context) {
}
func SetModelCount(ctx *context.Context) {
+ isQueryPrivate := isQueryPrivateModel(ctx)
repoId := ctx.Repo.Repository.ID
Type := -1
_, count, _ := models.QueryModel(&models.AiModelQueryOptions{
@@ -971,10 +964,12 @@ func SetModelCount(ctx *context.Context) {
Page: 1,
PageSize: 2,
},
- RepoID: repoId,
- Type: Type,
- New: MODEL_LATEST,
- Status: -1,
+ RepoID: repoId,
+ Type: Type,
+ New: MODEL_LATEST,
+ IsOnlyThisRepo: true,
+ Status: -1,
+ IsQueryPrivate: isQueryPrivate,
})
ctx.Data["MODEL_COUNT"] = count
}
@@ -1001,27 +996,87 @@ func isQueryRight(ctx *context.Context) bool {
}
}
+func isCanDownload(ctx *context.Context, task *models.AiModelManage) bool {
+ if ctx.User == nil {
+ return false
+ }
+ isCollaborator, err := ctx.Repo.Repository.IsCollaborator(ctx.User.ID)
+ if err != nil {
+ log.Info("query error.")
+ }
+ isTeamMember, err := ctx.Repo.Repository.IsInRepoTeam(ctx.User.ID)
+ if err != nil {
+ log.Info("query IsInRepoTeam error." + err.Error())
+ }
+ if ctx.User.IsAdmin || ctx.User.ID == task.UserId || isCollaborator || isTeamMember {
+ return true
+ }
+ if ctx.Repo.IsOwner() {
+ return true
+ }
+ if !task.IsPrivate {
+ return true
+ }
+ return false
+}
+
+func isQueryPrivateModel(ctx *context.Context) bool {
+ if ctx.User == nil {
+ return false
+ }
+ isCollaborator, err := ctx.Repo.Repository.IsCollaborator(ctx.User.ID)
+ if err != nil {
+ log.Info("query IsCollaborator error." + err.Error())
+ }
+ isTeamMember, err := ctx.Repo.Repository.IsInRepoTeam(ctx.User.ID)
+ if err != nil {
+ log.Info("query IsInRepoTeam error." + err.Error())
+ }
+ if ctx.User.IsAdmin || isCollaborator || isTeamMember {
+ return true
+ }
+ if ctx.Repo.IsOwner() {
+ return true
+ }
+ return false
+}
+
func isCanDelete(ctx *context.Context, modelUserId int64) bool {
if ctx.User == nil {
return false
}
- if ctx.User.IsAdmin || ctx.User.ID == modelUserId {
+ if ctx.User.ID == modelUserId {
+ return true
+ }
+ return isAdminRight(ctx)
+}
+
+func isAdminRight(ctx *context.Context) bool {
+ if ctx.User.IsAdmin {
return true
}
if ctx.Repo.IsOwner() {
return true
}
+ permission, err := models.GetUserRepoPermission(ctx.Repo.Repository, ctx.User)
+ if err != nil {
+ log.Error("GetUserRepoPermission failed:%v", err.Error())
+ return false
+ }
+ if permission.AccessMode >= models.AccessModeAdmin {
+ return true
+ }
return false
}
-func isOper(ctx *context.Context, modelUserId int64) bool {
+func isOperModifyOrDelete(ctx *context.Context, modelUserId int64) bool {
if ctx.User == nil {
return false
}
if ctx.User.IsAdmin || ctx.User.ID == modelUserId {
return true
}
- return false
+ return isAdminRight(ctx)
}
func ShowModelPageInfo(ctx *context.Context) {
@@ -1038,6 +1093,7 @@ func ShowModelPageInfo(ctx *context.Context) {
if pageSize <= 0 {
pageSize = setting.UI.IssuePagingNum
}
+ isQueryPrivate := isQueryPrivateModel(ctx)
repoId := ctx.Repo.Repository.ID
Type := -1
modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{
@@ -1045,10 +1101,12 @@ func ShowModelPageInfo(ctx *context.Context) {
Page: page,
PageSize: pageSize,
},
- RepoID: repoId,
- Type: Type,
- New: MODEL_LATEST,
- Status: -1,
+ RepoID: repoId,
+ Type: Type,
+ New: MODEL_LATEST,
+ IsOnlyThisRepo: true,
+ Status: -1,
+ IsQueryPrivate: isQueryPrivate,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
@@ -1057,8 +1115,9 @@ func ShowModelPageInfo(ctx *context.Context) {
userIds := make([]int64, len(modelResult))
for i, model := range modelResult {
- model.IsCanOper = isOper(ctx, model.UserId)
+ model.IsCanOper = isOperModifyOrDelete(ctx, model.UserId)
model.IsCanDelete = isCanDelete(ctx, model.UserId)
+ model.IsCanDownload = isCanDownload(ctx, model)
userIds[i] = model.UserId
}
@@ -1089,6 +1148,37 @@ func ModifyModel(id string, description string) error {
return err
}
+func ModifyModelPrivate(ctx *context.Context) {
+ id := ctx.Query("id")
+ isPrivate := ctx.QueryBool("isPrivate")
+ re := map[string]string{
+ "code": "-1",
+ }
+ task, err := models.QueryModelById(id)
+ if err != nil || task == nil {
+ re["msg"] = err.Error()
+ log.Error("no such model!", err.Error())
+ ctx.JSON(200, re)
+ return
+ }
+ if !isOperModifyOrDelete(ctx, task.UserId) {
+ re["msg"] = "No right to operation."
+ ctx.JSON(200, re)
+ return
+ }
+ err = models.ModifyModelPrivate(id, isPrivate)
+ if err == nil {
+ re["code"] = "0"
+ ctx.JSON(200, re)
+ log.Info("modify success.")
+ } else {
+ re["msg"] = err.Error()
+ ctx.JSON(200, re)
+ log.Info("Failed to modify.id=" + id + " isprivate=" + fmt.Sprint(isPrivate) + " error:" + err.Error())
+ }
+
+}
+
func ModifyModelInfo(ctx *context.Context) {
log.Info("modify model start.")
id := ctx.Query("id")
@@ -1102,7 +1192,7 @@ func ModifyModelInfo(ctx *context.Context) {
ctx.JSON(200, re)
return
}
- if !isOper(ctx, task.UserId) {
+ if !isOperModifyOrDelete(ctx, task.UserId) {
re["msg"] = "No right to operation."
ctx.JSON(200, re)
return
@@ -1112,6 +1202,7 @@ func ModifyModelInfo(ctx *context.Context) {
label := ctx.Query("label")
description := ctx.Query("description")
engine := ctx.QueryInt("engine")
+ isPrivate := ctx.QueryBool("isPrivate")
aimodels := models.QueryModelByName(name, task.RepoId)
if aimodels != nil && len(aimodels) > 0 {
if len(aimodels) == 1 {
@@ -1126,14 +1217,14 @@ func ModifyModelInfo(ctx *context.Context) {
return
}
}
- err = models.ModifyLocalModel(id, name, label, description, engine)
+ err = models.ModifyLocalModel(id, name, label, description, engine, isPrivate)
} else {
label := ctx.Query("label")
description := ctx.Query("description")
engine := task.Engine
name := task.Name
- err = models.ModifyLocalModel(id, name, label, description, int(engine))
+ err = models.ModifyLocalModel(id, name, label, description, int(engine), task.IsPrivate)
}
if err != nil {
@@ -1148,15 +1239,27 @@ func ModifyModelInfo(ctx *context.Context) {
func QueryModelListForPredict(ctx *context.Context) {
repoId := ctx.Repo.Repository.ID
+ page := ctx.QueryInt("page")
+ if page <= 0 {
+ page = -1
+ }
+ pageSize := ctx.QueryInt("pageSize")
+ if pageSize <= 0 {
+ pageSize = -1
+ }
+ isQueryPrivate := isQueryPrivateModel(ctx)
+ //IsOnlyThisRepo := ctx.QueryBool("isOnlyThisRepo")
modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{
ListOptions: models.ListOptions{
- Page: -1,
- PageSize: -1,
+ Page: page,
+ PageSize: pageSize,
},
- RepoID: repoId,
- Type: ctx.QueryInt("type"),
- New: -1,
- Status: 0,
+ RepoID: repoId,
+ Type: ctx.QueryInt("type"),
+ New: -1,
+ Status: 0,
+ IsOnlyThisRepo: true,
+ IsQueryPrivate: isQueryPrivate,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
@@ -1168,7 +1271,9 @@ func QueryModelListForPredict(ctx *context.Context) {
nameMap := make(map[string][]*models.AiModelManage)
for _, model := range modelResult {
- removeIpInfo(model)
+ model.TrainTaskInfo = ""
+ model.Accuracy = ""
+ //removeIpInfo(model)
if _, value := nameMap[model.Name]; !value {
models := make([]*models.AiModelManage, 0)
models = append(models, model)
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index d3d76f440..a23cd5462 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -670,6 +670,13 @@ func CloudBrainRestart(ctx *context.Context) {
break
}
+ if _, err := os.Stat(getOldJobPath(task)); err != nil {
+ log.Error("Can not find job minio path", err)
+ resultCode = "-1"
+ errorMsg = ctx.Tr("cloudbrain.result_cleared")
+ break
+ }
+
count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug))
if err != nil {
log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
@@ -704,6 +711,11 @@ func CloudBrainRestart(ctx *context.Context) {
})
}
+
+func getOldJobPath(task *models.Cloudbrain) string {
+ return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName
+}
+
func CloudBrainBenchMarkShow(ctx *context.Context) {
cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
}
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 01d2e2fa4..4e30e625d 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -2337,7 +2337,7 @@ func InferenceJobIndex(ctx *context.Context) {
tasks[i].ComputeResource = models.NPUResource
}
}
-
+ isQueryPrivate := isQueryPrivateModel(ctx)
repoId := ctx.Repo.Repository.ID
Type := -1
_, model_count, _ := models.QueryModel(&models.AiModelQueryOptions{
@@ -2345,10 +2345,12 @@ func InferenceJobIndex(ctx *context.Context) {
Page: 1,
PageSize: 2,
},
- RepoID: repoId,
- Type: Type,
- New: MODEL_LATEST,
- Status: 0,
+ RepoID: repoId,
+ Type: Type,
+ New: MODEL_LATEST,
+ IsOnlyThisRepo: true,
+ Status: 0,
+ IsQueryPrivate: isQueryPrivate,
})
ctx.Data["MODEL_COUNT"] = model_count
@@ -2417,7 +2419,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error {
return err
}
ctx.Data["config_list"] = configList.ParaConfigs
-
+ isQueryPrivate := isQueryPrivateModel(ctx)
repoId := ctx.Repo.Repository.ID
Type := -1
_, model_count, _ := models.QueryModel(&models.AiModelQueryOptions{
@@ -2425,10 +2427,12 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error {
Page: 1,
PageSize: 2,
},
- RepoID: repoId,
- Type: Type,
- New: MODEL_LATEST,
- Status: 0,
+ RepoID: repoId,
+ Type: Type,
+ New: MODEL_LATEST,
+ IsOnlyThisRepo: true,
+ Status: 0,
+ IsQueryPrivate: isQueryPrivate,
})
ctx.Data["MODEL_COUNT"] = model_count
ctx.Data["datasetType"] = models.TypeCloudBrainTwo
diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go
index 508addf75..a6de283a4 100755
--- a/routers/repo/user_data_analysis.go
+++ b/routers/repo/user_data_analysis.go
@@ -907,3 +907,9 @@ func QueryUserLoginInfo(ctx *context.Context) {
log.Info("writer exel error." + err.Error())
}
}
+
+func QueryUserAnnualReport(ctx *context.Context) {
+ log.Info("start to QueryUserAnnualReport ")
+ result := models.QueryUserAnnualReport(ctx.User.ID)
+ ctx.JSON(http.StatusOK, result)
+}
diff --git a/routers/repo/user_invitation.go b/routers/repo/user_invitation.go
index a2752a481..6e7207bce 100644
--- a/routers/repo/user_invitation.go
+++ b/routers/repo/user_invitation.go
@@ -49,9 +49,10 @@ func getInvitationDetailExcelHeader(ctx *context.Context) map[string]string {
excelHeader := make([]string, 0)
excelHeader = append(excelHeader, ctx.Tr("user.static.id"))
excelHeader = append(excelHeader, ctx.Tr("user.static.name"))
- excelHeader = append(excelHeader, ctx.Tr("user.static.srcUserId"))
+ excelHeader = append(excelHeader, ctx.Tr("user.static.email"))
excelHeader = append(excelHeader, ctx.Tr("user.static.phone"))
excelHeader = append(excelHeader, ctx.Tr("user.static.registdate"))
+ excelHeader = append(excelHeader, ctx.Tr("user.static.srcUserId"))
excelHeaderMap := make(map[string]string, 0)
var i byte
@@ -92,8 +93,7 @@ func writeInvitationDetailExcel(row int, xlsx *excelize.File, sheetName string,
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Name)
tmp = tmp + 1
-
- xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.SrcUserID)
+ xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Email)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.Phone)
@@ -101,7 +101,9 @@ func writeInvitationDetailExcel(row int, xlsx *excelize.File, sheetName string,
formatTime := userRecord.CreatedUnix.Format("2006-01-02 15:04:05")
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, formatTime[0:len(formatTime)-3])
+ tmp = tmp + 1
+ xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.SrcUserID)
}
func DownloadInvitationDetail(ctx *context.Context) {
@@ -413,6 +415,7 @@ func queryData(ctx *context.Context, startTime time.Time, endTime time.Time) {
invi.Name = tmpUser.Name
invi.Phone = tmpUser.PhoneNumber
invi.CreatedUnix = tmpUser.CreatedUnix
+ invi.Email = tmpUser.Email
} else {
invi.Name = "已注销"
}
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index e5c409a13..9a874c48e 100755
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -1262,6 +1262,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/delete_model_convert/:id", repo.DeleteModelConvert)
m.Post("/convert_stop/:id", repo.StopModelConvert)
m.Put("/modify_model", repo.ModifyModelInfo)
+ m.Put("/modify_model_status", repo.ModifyModelPrivate)
m.Get("/show_model", reqRepoModelManageReader, repo.ShowModelTemplate)
m.Get("/convert_model", reqRepoModelManageReader, repo.ConvertModelTemplate)
m.Get("/show_model_info", repo.ShowModelInfo)
diff --git a/routers/user/Invitation.go b/routers/user/Invitation.go
index 8491390b2..0eb8ae2f4 100644
--- a/routers/user/Invitation.go
+++ b/routers/user/Invitation.go
@@ -63,7 +63,7 @@ func InviationTpl(ctx *context.Context) {
ctx.HTML(200, tplInvitation)
}
-func RegisteUserByInvitaionCode(invitationcode string, newUserId int64, newPhoneNumber string) error {
+func RegisteUserByInvitaionCode(invitationcode string, newUserId int64, newPhoneNumber string, email string) error {
user := parseInvitaionCode(invitationcode)
if user == nil {
return errors.New("The invitated user not existed.")
@@ -85,6 +85,7 @@ func RegisteUserByInvitaionCode(invitationcode string, newUserId int64, newPhone
SrcUserID: user.ID,
UserID: newUserId,
Phone: newPhoneNumber,
+ Email: email,
}
err := models.InsertInvitaion(invitation)
diff --git a/routers/user/auth.go b/routers/user/auth.go
index 3d74b6ddd..5314571d2 100755
--- a/routers/user/auth.go
+++ b/routers/user/auth.go
@@ -1368,7 +1368,7 @@ func SignUpPost(ctx *context.Context, cpt *captcha.Captcha, form auth.RegisterFo
log.Info("enter here, and form.InvitaionCode =" + invitationCode)
if invitationCode != "" {
- RegisteUserByInvitaionCode(invitationCode, u.ID, u.PhoneNumber)
+ RegisteUserByInvitaionCode(invitationCode, u.ID, u.PhoneNumber, u.Email)
}
err := models.AddEmailAddress(&models.EmailAddress{
diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go
new file mode 100644
index 000000000..44613ee3c
--- /dev/null
+++ b/services/cloudbrain/clear.go
@@ -0,0 +1,151 @@
+package cloudbrain
+
+import (
+ "io/ioutil"
+ "os"
+ "sort"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/storage"
+)
+
+func ClearCloudbrainResultSpace() {
+ log.Info("clear cloudbrain one result space begin.")
+ if !setting.ClearStrategy.Enabled{
+ return
+ }
+
+ tasks, err := models.GetCloudBrainOneStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
+ if err != nil {
+ log.Warn("Failed to get cloudbrain, clear result failed.", err)
+ return
+ }
+ debugTasks, err := models.GetCloudBrainOneStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize)
+ if err != nil {
+ log.Warn("Failed to get debug cloudbrain.", err)
+
+ }
+ tasks=append(tasks,debugTasks...)
+
+ if err != nil {
+ log.Warn("Failed to get cloudbrain, clear result failed.", err)
+ return
+ }
+ var ids []int64
+ for _, task := range tasks {
+ err := DeleteCloudbrainOneJobStorage(task.JobName)
+ if err == nil {
+ log.Info("clear job in cloudbrain table:"+task.JobName)
+ ids = append(ids, task.ID)
+ }
+ }
+
+ err = models.UpdateCloudBrainRecordsCleared(ids)
+ if err != nil {
+ log.Warn("Failed to set cloudbrain cleared status", err)
+ }
+ //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话
+ if len(tasks) < setting.ClearStrategy.BatchSize+setting.ClearStrategy.DebugJobSize {
+ clearLocalHistoryTrashFile()
+ clearMinioHistoryTrashFile()
+
+ }
+ log.Info("clear cloudbrain one result space end.")
+
+}
+
+func clearMinioHistoryTrashFile() {
+ JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix
+
+ miniofiles, err := ioutil.ReadDir(JobRealPrefix)
+
+ processCount := 0
+ if err != nil {
+ log.Warn("Can not browser minio job path.")
+ } else {
+ SortModTimeAscend(miniofiles)
+ for _, file := range miniofiles {
+
+ if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
+
+ has,err:=models.IsCloudbrainExistByJobName(file.Name())
+ if err==nil && !has {
+ dirPath := setting.CBCodePathPrefix + file.Name() + "/"
+ log.Info("clear job in minio trash:" + file.Name())
+ storage.Attachments.DeleteDir(dirPath)
+ processCount++
+ }
+ if processCount == setting.ClearStrategy.BatchSize {
+ break
+ }
+ } else {
+ break
+ }
+
+ }
+
+ }
+}
+
+func clearLocalHistoryTrashFile() {
+ files, err := ioutil.ReadDir(setting.JobPath)
+ processCount := 0
+ if err != nil {
+ log.Warn("Can not browser local job path.")
+ } else {
+ SortModTimeAscend(files)
+ for _, file := range files {
+ //清理n天前的历史垃圾数据，清理job目录
+ if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
+ has,err:=models.IsCloudbrainExistByJobName(file.Name())
+ if err==nil && !has{
+ os.RemoveAll(setting.JobPath + file.Name())
+ log.Info("clear job in local trash:"+file.Name())
+ processCount++
+ }
+ if processCount == setting.ClearStrategy.BatchSize {
+ break
+ }
+ } else {
+ break
+ }
+
+ }
+
+ }
+
+}
+
+func SortModTimeAscend(files []os.FileInfo) {
+ sort.Slice(files, func(i, j int) bool {
+ return files[i].ModTime().Before(files[j].ModTime())
+ })
+}
+
+func DeleteCloudbrainOneJobStorage(jobName string) error {
+
+ if jobName==""{
+ return nil
+ }
+ //delete local
+ localJobPath := setting.JobPath + jobName
+ err := os.RemoveAll(localJobPath)
+ if err != nil {
+ log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
+ }
+
+ dirPath := setting.CBCodePathPrefix + jobName + "/"
+ err1 := storage.Attachments.DeleteDir(dirPath)
+
+ if err1 != nil {
+ log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
+ }
+ if err == nil {
+ err = err1
+ }
+
+ return err
+}
diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go
index 8f4182d87..5070d7c1e 100644
--- a/services/cloudbrain/resource/resource_specification.go
+++ b/services/cloudbrain/resource/resource_specification.go
@@ -246,10 +246,10 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S
return nil, err
}
//filter exclusive specs
- specs := filterExclusiveSpecs(r, userId)
+ specs := models.FilterExclusiveSpecs(r, userId)
//distinct by sourceSpecId
- specs = distinctSpecs(specs)
+ specs = models.DistinctSpecs(specs)
return specs, err
}
@@ -265,50 +265,6 @@ func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*api
return result, nil
}
-func filterExclusiveSpecs(r []*models.Specification, userId int64) []*models.Specification {
- specs := make([]*models.Specification, 0, len(r))
- specMap := make(map[int64]string, 0)
- for i := 0; i < len(r); i++ {
- spec := r[i]
- if _, has := specMap[spec.ID]; has {
- continue
- }
- if !spec.IsExclusive {
- specs = append(specs, spec)
- specMap[spec.ID] = ""
- continue
- }
- orgs := strings.Split(spec.ExclusiveOrg, ";")
- for _, org := range orgs {
- isMember, _ := models.IsOrganizationMemberByOrgName(org, userId)
- if isMember {
- specs = append(specs, spec)
- specMap[spec.ID] = ""
- break
- }
- }
- }
- return specs
-}
-
-func distinctSpecs(r []*models.Specification) []*models.Specification {
- specs := make([]*models.Specification, 0, len(r))
- sourceSpecIdMap := make(map[string]string, 0)
- for i := 0; i < len(r); i++ {
- spec := r[i]
- if spec.SourceSpecId == "" {
- specs = append(specs, spec)
- continue
- }
- if _, has := sourceSpecIdMap[spec.SourceSpecId]; has {
- continue
- }
- specs = append(specs, spec)
- sourceSpecIdMap[spec.SourceSpecId] = ""
- }
- return specs
-}
-
func GetAndCheckSpec(userId int64, specId int64, opts models.FindSpecsOptions) (*models.Specification, error) {
if specId == 0 {
return nil, nil
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index 69bec19e7..46534e4de 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -647,6 +647,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+