Merge remote-tracking branch 'origin/V20220428' into zouap_static

models/cloudbrain.go

@@ -1631,12 +1631,12 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
 	var err error
 	condition := "cloudbrain.user_id = `user`.id"
 	if len(opts.Keyword) == 0 {
-		count, err = sess.Where(cond).Count(new(Cloudbrain))
+		count, err = sess.Unscoped().Where(cond).Count(new(Cloudbrain))
 	} else {
 		lowerKeyWord := strings.ToLower(opts.Keyword)
 
 		cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord}))
-		count, err = sess.Table(&Cloudbrain{}).Where(cond).
+		count, err = sess.Table(&Cloudbrain{}).Unscoped().Where(cond).
 			Join("left", "`user`", condition).Count(new(CloudbrainInfo))
 
 	}

modules/repository/hooks.go

@@ -19,7 +19,11 @@ import (
 	"xorm.io/builder"
 )
 
-func getHookTemplates() (hookNames, hookTpls, giteaHookTpls []string) {
+const (
+	SIZE_LIMIT_SCRIPT_NAME = "size_limit"
+)
+
+func getHookTemplates() (hookNames, hookTpls, giteaHookTpls, sizeLimitTpls []string) {
 	hookNames = []string{"pre-receive", "update", "post-receive"}
 	hookTpls = []string{
 		fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" && test -f \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType),
@@ -31,6 +35,11 @@ func getHookTemplates() (hookNames, hookTpls, giteaHookTpls []string) {
 		fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' update $1 $2 $3\n", setting.ScriptType, setting.AppPath, setting.CustomConf),
 		fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' post-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf),
 	}
+	sizeLimitTpls = []string{
+		fmt.Sprintf("#!/usr/bin/env %s\n\n\nset -o pipefail\n\nreadonly DEFAULT_FILE_MAXSIZE_MB=\"30\" \nreadonly CONFIG_NAME=\"hooks.maxfilesize\"\nreadonly NULLSHA=\"0000000000000000000000000000000000000000\"\nreadonly EXIT_SUCCESS=0\nreadonly EXIT_FAILURE=1\nreadonly DEFAULT_REPO_MAXSIZE_MB=\"1024\" \nreadonly CHECK_FLAG_ON=0\n\n\nstatus=\"$EXIT_SUCCESS\"\n\nfunction readINI()\n{\n FILENAME='%s'; SECTION=$1; KEY=$2\n RESULT=`awk -F '=' '/\\['$SECTION'\\]/{a=1}a==1&&$1~/'$KEY'/{print $2;exit}' $FILENAME`\n echo $RESULT\n}\n\n# skip this hook entirely if shell check is not open\ncheck_flag=$(readINI 'repository.upload' 'SHELL_FLAG')\nif [[ $check_flag != $CHECK_FLAG_ON ]]; then\nexit $EXIT_SUCCESS\nfi\n\n\n#######################################\n# check the file max size limit\n#######################################\n\n# get the maximum filesize configured for this repository or the default\nfunction get_file_maxsize() {\n  local value;\n  value=$(readINI 'repository.upload' 'FILE_MAX_SIZE')\n  if [[ \"$?\" != $EXIT_SUCCESS ]] || [[ -z \"$value\" ]]; then\n    echo \"$DEFAULT_FILE_MAXSIZE_MB\"\n    return \"$EXIT_SUCCESS\"\n  fi\n  echo \"$value\"\n  return \"$EXIT_SUCCESS\"\n}\n\n# get maximum filesize (from repository-specific config)\nmaxsize_mb=\"$(get_file_maxsize)\"\n\nif [[ \"$?\" != $EXIT_SUCCESS ]]; then\necho \"failed to get ${CONFIG_NAME} from config\"\nexit \"$EXIT_FAILURE\"\nfi\n\npush_size=\"0\"\n# read lines from stdin (format: \"<oldref> <newref> <refname>\\n\")\nwhile read oldref newref refname; do\n# skip branch deletions\nif [[ \"$newref\" == \"$NULLSHA\" ]]; then\n  continue\nfi\n\n# find large objects\n# check all objects from $oldref (possible $NULLSHA) to $newref, but\n# skip all objects that have already been accepted (i.e. are referenced by\n# another branch or tag).\n\nif [[ \"$oldref\" == \"$NULLSHA\" ]]; then\n  target=\"$newref\"\nelse\n  target=\"${oldref}..${newref}\"\nfi\nmaxsize=`expr $maxsize_mb \\* 1048576` \n\n# find objects in this push_size\n# print like:\n# 08da8e2ab9ae4095bf94dd71ac913132b880b463 commit 214\n# 43e993b768ede5740e8c65de2ed6edec25053ea1 tree 185\n# 4476971d76569039df7569af1b8d03c288f6b193 blob 20167318 b0417e6593a1.zip\nfiles=\"$(git rev-list --objects \"$target\" --tags=\\* | \\\n  git cat-file $'--batch-check=%%(objectname) %%(objecttype) %%(objectsize) %%(rest)')\"\n  \nif [[ \"$?\" != $EXIT_SUCCESS ]]; then\n  echo \"failed to check for large files in ref ${refname}\"\n  continue\nfi\n\n# rewrite IFS to seperate line in $files\nIFS=$'\\n'\nfor file in $files; do\n  # if don't unset IFS,temp_array=(${file}) will get error answer\n  unset IFS\n  temp_array=(${file})\n  # add all commit files size\n  push_size=`expr $push_size + ${temp_array[2]}`\n  if [[ ${temp_array[2]} -gt $maxsize ]]; then\n\t if [[ \"$status\" == $EXIT_SUCCESS ]]; then\n\t\techo -e \"Error: Your push was rejected because it contains files larger than $(numfmt --to=iec \"$maxsize_mb\") Mb\"\n\t\techo \"oversize files:\"\n\t\tstatus=\"$EXIT_FAILURE\"\n\t fi\n\t  echo -e \"\\033[31m- ${temp_array[3]} \\033[0m (ref: ${refname}) \"\n  fi\ndone\n\nif [[ \"$status\" != $EXIT_SUCCESS ]]; then\n\texit \"$status\"\nfi\n\ndone\n\n#######################################\n# check the repo max size limit\n#######################################\nif [[ $push_size -eq \"0\" ]]; then\n\texit $EXIT_SUCCESS\nfi\n\nfunction get_repo_maxsize() {\n  local value;\n  value=$(readINI 'repository' 'REPO_MAX_SIZE')\n  if [[ \"$?\" != $EXIT_SUCCESS ]] || [[ -z \"$value\" ]]; then\n    echo \"$DEFAULT_FILE_MAXSIZE\"\n    return \"$EXIT_SUCCESS\"\n  fi\n  echo \"$value\"\n  return \"$EXIT_SUCCESS\"\n}\n\n\nsizelimit_mb=\"$(get_repo_maxsize)\"\nlet sizelimit_b=$sizelimit_mb*1024*1024\n\n# repo size at here means the size of repo directory in server \nstr=`du -sb .`\narr=($str)\nreposize_b=${arr[0]}\n\ntotal=`expr $push_size + $reposize_b`\n\nif [ $total -gt $sizelimit_b ]; then\n    echo \"Error: Your push was rejected because the repository size is large than $sizelimit_mb Mb\"\n    exit $EXIT_FAILURE\nfi\n\n\nexit $EXIT_SUCCESS\n", setting.ScriptType, setting.CustomConf),
+		fmt.Sprintf(""),
+		fmt.Sprintf(""),
+	}
 	return
 }
 
@@ -41,7 +50,7 @@ func CreateDelegateHooks(repoPath string) error {
 
 // createDelegateHooks creates all the hooks scripts for the repo
 func createDelegateHooks(repoPath string) (err error) {
-	hookNames, hookTpls, giteaHookTpls := getHookTemplates()
+	hookNames, hookTpls, giteaHookTpls, sizeLimitTpls := getHookTemplates()
 	hookDir := filepath.Join(repoPath, "hooks")
 
 	for i, hookName := range hookNames {
@@ -74,8 +83,26 @@ func createDelegateHooks(repoPath string) (err error) {
 		if err = ensureExecutable(newHookPath); err != nil {
 			return fmt.Errorf("Unable to set %s executable. Error %v", oldHookPath, err)
 		}
+
+		if err = writeHookTpl(generateHookScriptPath(hookDir, hookName, SIZE_LIMIT_SCRIPT_NAME), sizeLimitTpls[i]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func writeHookTpl(hookPath, content string) error {
+	if content == "" {
+		return nil
+	}
+	if err := ioutil.WriteFile(hookPath, []byte(content), 0777); err != nil {
+		return fmt.Errorf("write new hook file '%s': %v", hookPath, err)
 	}
 
+	if err := ensureExecutable(hookPath); err != nil {
+		return fmt.Errorf("Unable to set %s executable. Error %v", hookPath, err)
+	}
 	return nil
 }
 
@@ -101,7 +128,7 @@ func ensureExecutable(filename string) error {
 
 // CheckDelegateHooks checks the hooks scripts for the repo
 func CheckDelegateHooks(repoPath string) ([]string, error) {
-	hookNames, hookTpls, giteaHookTpls := getHookTemplates()
+	hookNames, hookTpls, giteaHookTpls, sizeLimitTpls := getHookTemplates()
 
 	hookDir := filepath.Join(repoPath, "hooks")
 	results := make([]string, 0, 10)
@@ -146,10 +173,34 @@ func CheckDelegateHooks(repoPath string) ([]string, error) {
 		if !checkExecutable(newHookPath) {
 			results = append(results, fmt.Sprintf("new hook file %s is not executable", newHookPath))
 		}
+		if err = checkHookFile(generateHookScriptPath(hookDir, hookName, SIZE_LIMIT_SCRIPT_NAME), sizeLimitTpls[i], results); err != nil {
+			return results, err
+		}
 	}
 	return results, nil
 }
 
+func generateHookScriptPath(hookDir, hookName, fileName string) string {
+	return filepath.Join(hookDir, hookName+".d", fileName)
+}
+
+func checkHookFile(filePath, tpl string, results []string) error {
+	if tpl == "" {
+		return nil
+	}
+	contents, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+	if string(contents) != tpl {
+		results = append(results, fmt.Sprintf("old hook file %s is out of date", filePath))
+	}
+	if !checkExecutable(filePath) {
+		results = append(results, fmt.Sprintf("old hook file %s is not executable", filePath))
+	}
+	return nil
+}
+
 // SyncRepositoryHooks rewrites all repositories' pre-receive, update and post-receive hooks
 // to make sure the binary and custom conf path are up-to-date.
 func SyncRepositoryHooks(ctx context.Context) error {

routers/api/v1/repo/cloudbrain_dashboard.go

@@ -23,7 +23,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) {
 	_, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{
 		ListOptions: models.ListOptions{
 			Page:     page,
-			PageSize: 1,
+			PageSize: pageSize,
 		},
 		Type:         models.TypeCloudBrainAll,
 		NeedRepoInfo: false,

routers/user/home.go

@@ -804,6 +804,7 @@ func Cloudbrains(ctx *context.Context) {
 	repos, _, err := models.SearchRepository(&models.SearchRepoOptions{
 		Actor:   ctx.User,
 		OwnerID: ctxUser.ID,
+		Private: true,
 	})
 	if err != nil {
 		ctx.ServerError("SearchRepository", err)

templates/admin/cloudbrain/list.tmpl

@@ -162,9 +162,9 @@
                                         {{end}}
                                     </div>
                                     <!-- 删除任务 -->
-                                    <form class="ui compact buttons" id="delForm-{{$JobID}}" action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}/train-job{{end}}/{{$JobID}}/del?isadminpage=true' method="post">
+                                    <form class="ui compact buttons" id="delForm-{{$JobID}}" action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}/train-job{{end}}/{{$JobID}}/del?isadminpage=true' method="post">
                                         {{$.CsrfTokenHtml}}
-                                        <a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}" data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?isadminpage=true"  data-version="{{.VersionName}}" class="ui basic ai_delete blue button" style="border-radius: .28571429rem;">
+                                        <a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}" data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?isadminpage=true"  data-version="" class="ui basic ai_delete blue button" style="border-radius: .28571429rem;">
                                             {{$.i18n.Tr "repo.delete"}}
                                         </a>
                                     </form>

templates/user/dashboard/cloudbrains.tmpl

@@ -147,9 +147,9 @@
                                         {{end}}
                                     </div>
                                     <!-- 删除任务 -->
-                                    <form class="ui compact buttons" id="delForm-{{$JobID}}" action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true' method="post">
+                                    <form class="ui compact buttons" id="delForm-{{$JobID}}" action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/train-job{{else}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true' method="post">
                                         {{$.CsrfTokenHtml}}
-                                        <a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}" data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?ishomepage=true"  data-version="{{.VersionName}}" class="ui basic ai_delete blue button" style="border-radius: .28571429rem;">
+                                        <a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}" data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?ishomepage=true"  data-version="" class="ui basic ai_delete blue button" style="border-radius: .28571429rem;">
                                             {{$.i18n.Tr "repo.delete"}}
                                         </a>
                                     </form>