aiforge/routers/api/v1/repo/cloudbrain.go

// Copyright 2016 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"bufio"
	"encoding/json"
	"io"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/notification"

	"code.gitea.io/gitea/modules/setting"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/cloudbrain"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/modelarts"
	"code.gitea.io/gitea/modules/storage"
	routerRepo "code.gitea.io/gitea/routers/repo"
)

// cloudbrain get job task by jobid
func GetCloudbrainTask(ctx *context.APIContext) {
	// swagger:operation GET /repos/{owner}/{repo}/cloudbrain/{jobid} cloudbrain jobTask
	// ---
	// summary: Get a single task
	// produces:
	// - application/json
	// parameters:
	// - name: owner
	//   in: path
	//   description: owner of the repo
	//   type: string
	//   required: true
	// - name: repo
	//   in: path
	//   description: name of the repo
	//   type: string
	//   required: true
	// - name: jobid
	//   in: path
	//   description: id of cloudbrain jobid
	//   type: string
	//   required: true
	// responses:
	//   "200":

	//     "$ref": "#/responses/Label"

	var (
		err error
	)

	ID := ctx.Params(":id")
	job, err := models.GetCloudbrainByID(ID)
	if err != nil {
		ctx.NotFound(err)
		log.Error("GetCloudbrainByID failed:", err)
		return
	}
	jobResult, err := cloudbrain.GetJob(job.JobID)
	if err != nil {
		ctx.NotFound(err)
		log.Error("GetJob failed:", err)
		return
	}
	result, _ := models.ConvertToJobResultPayload(jobResult.Payload)
	if err != nil {
		ctx.NotFound(err)
		log.Error("ConvertToJobResultPayload failed:", err)
		return
	}
	oldStatus := job.Status
	job.Status = result.JobStatus.State
	taskRoles := result.TaskRoles
	taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
	if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
		job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
		job.ContainerID = taskRes.TaskStatuses[0].ContainerID
		job.Status = taskRes.TaskStatuses[0].State
	}

	if result.JobStatus.State != string(models.JobWaiting) {
		models.ParseAndSetDurationFromCloudBrainOne(result, job)
		if oldStatus != job.Status {
			notification.NotifyChangeCloudbrainStatus(job, oldStatus)
		}
		err = models.UpdateJob(job)
		if err != nil {
			log.Error("UpdateJob failed:", err)
		}
	}

	ctx.JSON(http.StatusOK, map[string]interface{}{
		"ID":            ID,
		"JobName":       result.Config.JobName,
		"JobStatus":     result.JobStatus.State,
		"SubState":      result.JobStatus.SubState,
		"CreatedTime":   time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"),
		"CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"),
		"JobDuration":   job.TrainJobDuration,
	})

}

func GetCloudBrainInferenceJob(ctx *context.APIContext) {

	jobID := ctx.Params(":jobid")
	job, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		ctx.NotFound(err)
		return
	}
	jobResult, err := cloudbrain.GetJob(job.JobID)
	if err != nil {
		ctx.NotFound(err)
		log.Error("GetJob failed:", err)
		return
	}
	result, err := models.ConvertToJobResultPayload(jobResult.Payload)
	if err != nil {
		ctx.NotFound(err)
		log.Error("ConvertToJobResultPayload failed:", err)
		return
	}
	oldStatus := job.Status
	job.Status = result.JobStatus.State
	if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
		taskRoles := result.TaskRoles
		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))

		job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
		job.ContainerID = taskRes.TaskStatuses[0].ContainerID
		job.Status = taskRes.TaskStatuses[0].State
	}

	if result.JobStatus.State != string(models.JobWaiting) {
		models.ParseAndSetDurationFromCloudBrainOne(result, job)
		if oldStatus != job.Status {
			notification.NotifyChangeCloudbrainStatus(job, oldStatus)
		}
		err = models.UpdateJob(job)
		if err != nil {
			log.Error("UpdateJob failed:", err)
		}
	}

	ctx.JSON(http.StatusOK, map[string]interface{}{
		"JobID":       jobID,
		"JobStatus":   job.Status,
		"JobDuration": job.TrainJobDuration,
	})

}

func DelCloudBrainJob(ctx *context.APIContext) {
	jobID := ctx.Params(":jobid")

	errStr := cloudbrain.DelCloudBrainJob(jobID)

	if errStr != "" {
		ctx.JSON(http.StatusOK, map[string]interface{}{
			"Message":     ctx.Tr(errStr),
			"VersionName": "1",
			"Code":        1,
		})
	} else {
		ctx.JSON(http.StatusOK, map[string]interface{}{
			"Message":     "",
			"VersionName": "1",
			"Code":        0,
		})
	}

}

func InferencJobResultList(ctx *context.APIContext) {
	jobID := ctx.Params(":jobid")
	parentDir := ctx.Query("parentDir")
	dirArray := strings.Split(parentDir, "/")

	task, err := models.GetCloudbrainByJobID(jobID)
	if err != nil {
		log.Error("get cloud brain err:", err)
		ctx.ServerError("get cloud brain information failed:", err)

	}

	//get dirs
	dirs, err := routerRepo.GetResultDirs(task.JobName, parentDir)
	if err != nil {
		log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("GetModelDirs failed:", err)
		return
	}

	var fileInfos []storage.FileInfo
	err = json.Unmarshal([]byte(dirs), &fileInfos)
	if err != nil {
		log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("json.Unmarshal failed:", err)
		return
	}

	for i, fileInfo := range fileInfos {
		temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
		fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
	}

	sort.Slice(fileInfos, func(i, j int) bool {
		return fileInfos[i].ModTime > fileInfos[j].ModTime
	})

	ctx.JSON(http.StatusOK, map[string]interface{}{
		"JobID":            jobID,
		"StatusOK":         0,
		"Path":             dirArray,
		"Dirs":             fileInfos,
		"task":             task,
		"PageIsCloudBrain": true,
	})

}

func GetCloudbrainModelConvertTask(ctx *context.APIContext) {
	var (
		err error
	)
	ID := ctx.Params(":id")
	job, err := models.QueryModelConvertById(ID)
	if err != nil {
		ctx.NotFound(err)
		log.Error("GetCloudbrainByID failed:", err)
		return
	}
	if job.IsGpuTrainTask() {
		jobResult, err := cloudbrain.GetJob(job.CloudBrainTaskId)
		if err != nil {
			ctx.NotFound(err)
			log.Error("GetJob failed:", err)
			return
		}
		result, _ := models.ConvertToJobResultPayload(jobResult.Payload)
		if err != nil {
			ctx.NotFound(err)
			log.Error("ConvertToJobResultPayload failed:", err)
			return
		}

		job.Status = result.JobStatus.State
		taskRoles := result.TaskRoles
		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
		if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) {
			job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
			job.ContainerID = taskRes.TaskStatuses[0].ContainerID
			job.Status = taskRes.TaskStatuses[0].State
		}

		if result.JobStatus.State != string(models.JobWaiting) {
			models.ModelComputeAndSetDuration(job, result)
			err = models.UpdateModelConvert(job)
			if err != nil {
				log.Error("UpdateJob failed:", err)
			}
		}
		ctx.JSON(http.StatusOK, map[string]interface{}{
			"ID":            ID,
			"JobName":       result.Config.JobName,
			"JobStatus":     result.JobStatus.State,
			"SubState":      result.JobStatus.SubState,
			"CreatedTime":   time.Unix(result.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05"),
			"CompletedTime": time.Unix(result.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05"),
		})
	} else {

		result, err := modelarts.GetTrainJob(job.CloudBrainTaskId, job.ModelArtsVersionId)
		if err != nil {
			log.Error("get modelart job failed:", err)
			ctx.NotFound(err)
			return
		}

		job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
		job.RunTime = result.Duration / 1000
		job.TrainJobDuration = models.ConvertDurationToStr(job.RunTime)
		err = models.UpdateModelConvert(job)
		if err != nil {
			log.Error("UpdateJob failed:", err)
		}

		ctx.JSON(http.StatusOK, map[string]interface{}{
			"ID":        ID,
			"JobStatus": job.Status,
		})

	}

}

func CloudbrainGetLogByJobId(jobId string, jobName string) map[string]interface{} {
	var hits []models.Hits
	result, err := cloudbrain.GetJobLog(jobId)
	if err != nil {
		log.Error("GetJobLog failed: %v", err)
		return nil
	}
	hits = result.Hits.Hits

	//if the size equal page_size, then take the scroll_id to get all log and delete the scroll_id(the num of scroll_id is limited)
	if len(result.Hits.Hits) >= cloudbrain.LogPageSize {
		for {
			resultNext, err := cloudbrain.GetJobAllLog(result.ScrollID)
			if err != nil {
				log.Error("GetJobAllLog failed: %v", err)
			} else {
				for _, hit := range resultNext.Hits.Hits {
					hits = append(hits, hit)
				}
			}

			if len(resultNext.Hits.Hits) < cloudbrain.LogPageSize {
				log.Info("get all log already")
				break
			}
		}
	}

	cloudbrain.DeleteJobLogToken(result.ScrollID)

	sort.Slice(hits, func(i, j int) bool {
		return hits[i].Sort[0] < hits[j].Sort[0]
	})

	var content string
	for _, log := range hits {
		content += log.Source.Message + "\n"
	}

	return map[string]interface{}{
		"JobName": jobName,
		"Content": content,
	}

}

func CloudbrainForModelConvertGetLog(ctx *context.Context) {
	ID := ctx.Params(":id")
	job, err := models.QueryModelConvertById(ID)
	if err != nil {
		log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	result := CloudbrainGetLogByJobId(job.CloudBrainTaskId, job.Name)
	if result == nil {
		log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
		ctx.ServerError(err.Error(), err)
		return
	}
	ctx.JSON(http.StatusOK, result)
}

func CloudbrainDownloadLogFile(ctx *context.Context) {
	ID := ctx.Params(":id")
	job, err := models.GetCloudbrainByID(ID)
	if err != nil {
		log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	prefix := "/" + setting.CBCodePathPrefix + job.JobName + "/model"
	files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
	if err != nil {
		log.Error("query cloudbrain model failed: %v", err)
		return
	}
	fileName := ""
	for _, file := range files {
		if strings.HasSuffix(file.FileName, "log.txt") {
			fileName = file.FileName
			break
		}
	}
	if fileName != "" {
		url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName)
		if err != nil {
			log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
			ctx.ServerError("Get minio get SignedUrl failed", err)
			return
		}
		http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
	}
}

func CloudbrainGetLog(ctx *context.Context) {
	ID := ctx.Params(":id")
	startLine := ctx.QueryInt("base_line")
	lines := ctx.QueryInt("lines")
	endLine := startLine + lines
	order := ctx.Query("order")
	if order == "asc" {
		endLine = startLine
		startLine = endLine - lines
		if startLine < 0 {
			startLine = 0
		}
	}
	job, err := models.GetCloudbrainByID(ID)
	if err != nil {
		log.Error("GetCloudbrainByJobName failed: %v", err, ctx.Data["MsgID"])
		ctx.ServerError(err.Error(), err)
		return
	}
	result := getLogFromModelDir(job.JobName, startLine, endLine)
	if result == nil {
		log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	re := map[string]interface{}{
		"JobID":          ID,
		"LogFileName":    result["FileName"],
		"StartLine":      startLine,
		"EndLine":        result["endLine"],
		"Content":        result["Content"],
		"Lines":          result["lines"],
		"CanLogDownload": result["FileName"] != "",
	}
	//result := CloudbrainGetLogByJobId(job.JobID, job.JobName)

	ctx.JSON(http.StatusOK, re)
}

func getLogFromModelDir(jobName string, startLine int, endLine int) map[string]interface{} {
	prefix := "/" + setting.CBCodePathPrefix + jobName + "/model"
	files, err := storage.GetOneLevelAllObjectUnderDirMinio(setting.Attachment.Minio.Bucket, prefix, "")
	if err != nil {
		log.Error("query cloudbrain model failed: %v", err)
		return nil
	}

	re := ""
	fileName := ""
	count := 0
	fileEndLine := endLine
	for _, file := range files {
		if strings.HasSuffix(file.FileName, "log.txt") {
			fileName = file.FileName
			path := storage.GetMinioPath(jobName+"/model/", file.FileName)
			log.Info("path=" + path)
			reader, err := os.Open(path)
			defer reader.Close()
			if err == nil {
				r := bufio.NewReader(reader)
				for i := 0; i < endLine; i++ {
					line, error := r.ReadString('\n')
					log.Info("line=" + line)
					fileEndLine = i
					if error == io.EOF {
						log.Info("read file completed.")
						break
					}
					if error != nil {
						log.Info("read file error." + error.Error())
						break
					}
					if error == nil {
						if i >= startLine {
							re = re + line
							count++
						}
					}
				}
			} else {
				log.Info("error:" + err.Error())
			}
			break
		}
	}

	return map[string]interface{}{
		"JobName":  jobName,
		"Content":  re,
		"FileName": fileName,
		"lines":    count,
		"endLine":  fileEndLine,
	}
}

func CloudBrainModelConvertList(ctx *context.APIContext) {
	var (
		err error
	)

	ID := ctx.Params(":id")
	parentDir := ctx.Query("parentDir")
	dirArray := strings.Split(parentDir, "/")

	job, err := models.QueryModelConvertById(ID)
	if err != nil {
		log.Error("GetCloudbrainByJobID(%s) failed:%v", job.Name, err.Error())
		return
	}
	if job.IsGpuTrainTask() {
		//get dirs
		dirs, err := routerRepo.GetModelDirs(job.ID, parentDir)
		if err != nil {
			log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
			ctx.ServerError("GetModelDirs failed:", err)
			return
		}

		var fileInfos []storage.FileInfo
		err = json.Unmarshal([]byte(dirs), &fileInfos)
		if err != nil {
			log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
			ctx.ServerError("json.Unmarshal failed:", err)
			return
		}

		for i, fileInfo := range fileInfos {
			temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
			fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
		}

		sort.Slice(fileInfos, func(i, j int) bool {
			return fileInfos[i].ModTime > fileInfos[j].ModTime
		})

		ctx.JSON(http.StatusOK, map[string]interface{}{
			"JobID":            ID,
			"VersionName":      "",
			"StatusOK":         0,
			"Path":             dirArray,
			"Dirs":             fileInfos,
			"task":             job,
			"PageIsCloudBrain": true,
		})
	} else {
		var jobID = ctx.Params(":id")
		var versionName = "V0001"
		parentDir := ctx.Query("parentDir")
		dirArray := strings.Split(parentDir, "/")

		models, err := storage.GetObsListObject(job.ID, "output/", parentDir, versionName)
		if err != nil {
			log.Info("get TrainJobListModel failed:", err)
			ctx.ServerError("GetObsListObject:", err)
			return
		}

		ctx.JSON(http.StatusOK, map[string]interface{}{
			"JobID":            jobID,
			"VersionName":      versionName,
			"StatusOK":         0,
			"Path":             dirArray,
			"Dirs":             models,
			"task":             job,
			"PageIsCloudBrain": true,
		})
	}

}

func CloudBrainModelList(ctx *context.APIContext) {
	var (
		err error
	)

	var jobID = ctx.Params(":jobid")
	var versionName = ctx.Query("version_name")
	parentDir := ctx.Query("parentDir")
	dirArray := strings.Split(parentDir, "/")

	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
	if err != nil {
		log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
		return
	}

	//get dirs
	dirs, err := routerRepo.GetModelDirs(task.JobName, parentDir)
	if err != nil {
		log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("GetModelDirs failed:", err)
		return
	}

	var fileInfos []storage.FileInfo
	err = json.Unmarshal([]byte(dirs), &fileInfos)
	if err != nil {
		log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("json.Unmarshal failed:", err)
		return
	}

	for i, fileInfo := range fileInfos {
		temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
		fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
	}

	sort.Slice(fileInfos, func(i, j int) bool {
		return fileInfos[i].ModTime > fileInfos[j].ModTime
	})

	ctx.JSON(http.StatusOK, map[string]interface{}{
		"JobID":            jobID,
		"VersionName":      versionName,
		"StatusOK":         0,
		"Path":             dirArray,
		"Dirs":             fileInfos,
		"task":             task,
		"PageIsCloudBrain": true,
	})
}

type JobInfo struct {
	JobName    string `json:"job_name"`
	AiCenterId int    `json:"ai_center_id"`
}

func GetNewestJobs(ctx *context.APIContext) {
	idsC2Net, err := models.GetNewestJobsByAiCenter()
	if err != nil {
		log.Error("GetNewestJobsByAiCenter(%s) failed:%v", err.Error())
		return
	}

	idsCloudbrain, err := models.GetNewestJobsByType()
	if err != nil {
		log.Error("GetNewestJobsByType(%s) failed:%v", err.Error())
		return
	}

	ids := make([]int64, len(idsC2Net), cap(idsC2Net)*2)
	copy(ids, idsC2Net)
	for _, id := range idsCloudbrain {
		ids = append(ids, id)
	}

	jobs, err := models.GetCloudbrainByIDs(ids)
	if err != nil {
		log.Error("GetCloudbrainByIDs(%s) failed:%v", err.Error())
		return
	}

	jobInfos := make([]JobInfo, 0)
	for _, job := range jobs {
		var id int
		var content string
		switch job.Type {
		case models.TypeCloudBrainOne:
			id, content = getAICenterID("cloudbrain_one")
			if content == "" {
				log.Error("job(%s) has no match config info", job.DisplayJobName)
				continue
			}
		case models.TypeCloudBrainTwo:
			id, content = getAICenterID("cloudbrain_two")
			if content == "" {
				log.Error("job(%s) has no match config info", job.DisplayJobName)
				continue
			}
		case models.TypeC2Net:
			centerInfo := strings.Split(job.AiCenter, "+")
			if len(centerInfo) != 2 {
				log.Error("job(%s):ai_center(%s) is wrong", job.DisplayJobName, job.AiCenter)
				continue
			}
			id, content = getAICenterID(centerInfo[0])
			if content == "" {
				log.Error("job(%s) has no match config info", job.DisplayJobName)
				continue
			}
		default:
			log.Error("no match info")
			continue
		}

		jobInfos = append(jobInfos, JobInfo{
			JobName:    job.DisplayJobName,
			AiCenterId: id,
		})
	}

	ctx.JSON(http.StatusOK, jobInfos)
}

func GetAICenterInfo(ctx *context.APIContext) {
	if setting.C2NetInfos == nil {
		log.Error("C2NET_SEQUENCE is incorrect")
		return
	}

	ctx.JSON(http.StatusOK, setting.C2NetInfos.C2NetSqInfo)
}

func getAICenterID(name string) (int, string) {
	for _, info := range setting.C2NetInfos.C2NetSqInfo {
		if name == info.Name {
			return info.ID, info.Content
		}
	}

	return 0, ""
}