From a63a22ea1c984078bdcc2080f35dd5edd250e5eb Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Tue, 22 Mar 2022 17:17:55 +0800
Subject: [PATCH 01/13] #1654 add task duration

---
 models/cloudbrain.go              | 18 ++++++++++++++++--
 routers/api/v1/repo/cloudbrain.go |  5 +++++
 routers/repo/cloudbrain.go        | 24 +++++++++++++++++++++++-
 routers/repo/modelarts.go         |  5 +++++
 4 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index f501d8e91..a7a0cbd98 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -150,6 +150,19 @@ type Cloudbrain struct {
 	Repo                  *Repository `xorm:"-"`
 	BenchmarkTypeName     string      `xorm:"-"`
 	BenchmarkTypeRankLink string      `xorm:"-"`
+	StartTime             timeutil.TimeStamp
+	EndTime               timeutil.TimeStamp
+}
+
+func (task *Cloudbrain) ComputeDuration() int64 {
+	if task.StartTime == 0 || task.EndTime == 0 {
+		return 0
+	}
+	d := task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
+	if d < 0 {
+		d = 0
+	}
+	return d
 }
 
 type CloudbrainInfo struct {
@@ -1019,6 +1032,7 @@ type GetTrainJobResult struct {
 	NasShareAddr     string      `json:"nas_share_addr"`
 	DatasetName      string
 	ModelMetricList  string `json:"model_metric_list"` //列表里包含f1_score，recall，precision，accuracy，若有的话
+	StartTime        int64  `json:"start_time"`        //训练作业开始时间。
 }
 
 type GetTrainJobLogResult struct {
@@ -1327,13 +1341,13 @@ func GetCloudbrainByJobIDAndIsLatestVersion(jobID string, isLatestVersion string
 
 func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) {
 	cloudBrains := make([]*Cloudbrain, 0)
-	err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains)
+	err := x.Cols("job_id", "status", "type", "job_type", "version_id", "start_time").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains)
 	return cloudBrains, err
 }
 
 func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) {
 	cloudBrains := make([]*Cloudbrain, 0)
-	err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains)
+	err := x.Cols("job_id", "status", "type", "job_type", "version_id", "start_time").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains)
 	return cloudBrains, err
 }
 
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index f92259c3d..53a3d9545 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -6,6 +6,7 @@
 package repo
 
 import (
+	"code.gitea.io/gitea/modules/timeutil"
 	"net/http"
 	"sort"
 	"time"
@@ -77,6 +78,10 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 		job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
 		job.ContainerID = taskRes.TaskStatuses[0].ContainerID
 		job.Status = taskRes.TaskStatuses[0].State
+
+		if job.StartTime == 0 {
+			job.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
+		}
 	}
 
 	if result.JobStatus.State != string(models.JobWaiting) {
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 6e88b266d..86f111641 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -2,6 +2,7 @@ package repo
 
 import (
 	"bufio"
+	"code.gitea.io/gitea/modules/timeutil"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -380,6 +381,9 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) {
 			task.Status = taskRes.TaskStatuses[0].State
 			task.ContainerID = taskRes.TaskStatuses[0].ContainerID
 			task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
+			if task.StartTime == 0 {
+				task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
+			}
 			err = models.UpdateJob(task)
 			if err != nil {
 				ctx.Data["error"] = err.Error()
@@ -489,6 +493,8 @@ func CloudBrainStop(ctx *context.Context) {
 		}
 
 		task.Status = string(models.JobStopped)
+		task.EndTime = timeutil.TimeStampNow()
+		task.Duration = task.ComputeDuration()
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
@@ -582,6 +588,8 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
 		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
 	} else {
 		taskInfo.Status = string(models.JobStopped)
+		taskInfo.EndTime = timeutil.TimeStampNow()
+		taskInfo.Duration = taskInfo.ComputeDuration()
 		err = models.UpdateJob(taskInfo)
 		if err != nil {
 			log.Warn("UpdateJob failed", err)
@@ -953,6 +961,13 @@ func SyncCloudbrainStatus() {
 				task.Status = taskRes.TaskStatuses[0].State
 				if task.Status != string(models.JobWaiting) {
 					task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix()
+					if task.StartTime == 0 {
+						task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
+					}
+					//if task.Status == string(models.JobFailed) {
+					//	task.EndTime = timeutil.TimeStampNow()
+					//	task.Duration = task.ComputeDuration()
+					//}
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -973,6 +988,8 @@ func SyncCloudbrainStatus() {
 							continue
 						}
 						task.Status = string(models.JobStopped)
+						task.EndTime = timeutil.TimeStampNow()
+						task.Duration = task.ComputeDuration()
 						err = models.UpdateJob(task)
 						if err != nil {
 							log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -991,7 +1008,9 @@ func SyncCloudbrainStatus() {
 
 				if result != nil {
 					task.Status = result.Status
-
+					if task.StartTime == 0 {
+						task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
+					}
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1010,6 +1029,9 @@ func SyncCloudbrainStatus() {
 					task.Duration = result.Duration
 					task.TrainJobDuration = result.TrainJobDuration
 
+					if task.StartTime == 0 {
+						task.StartTime = timeutil.TimeStamp(result.StartTime)
+					}
 					if result.Duration != 0 {
 						task.TrainJobDuration = util.AddZero(result.Duration/3600000) + ":" + util.AddZero(result.Duration%3600000/60000) + ":" + util.AddZero(result.Duration%60000/1000)
 
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 9c670e203..6b5eeffbf 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -2,6 +2,7 @@ package repo
 
 import (
 	"archive/zip"
+	"code.gitea.io/gitea/modules/timeutil"
 	"encoding/json"
 	"errors"
 	"io"
@@ -408,6 +409,10 @@ func NotebookManage(ctx *context.Context) {
 		}
 
 		task.Status = res.Status
+		if task.Status == string(models.ModelArtsStopped) {
+			task.EndTime = timeutil.TimeStampNow()
+			task.Duration = task.ComputeDuration()
+		}
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"])

From e46903c41424b4d72bfb2d740e3c6001a9436489 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Tue, 22 Mar 2022 17:56:42 +0800
Subject: [PATCH 02/13] #1654 update

---
 routers/api/v1/repo/modelarts.go | 9 ++++++++-
 routers/repo/cloudbrain.go       | 4 ++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 893f2a32c..cc311c446 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -6,6 +6,7 @@
 package repo
 
 import (
+	"code.gitea.io/gitea/modules/timeutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -67,8 +68,14 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-
+	if job.StartTime == 0 {
+		job.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
+	}
 	job.Status = result.Status
+	if job.EndTime == 0 && job.Status == string(models.ModelArtsStopped) {
+		job.EndTime = timeutil.TimeStampNow()
+		job.Duration = job.ComputeDuration()
+	}
 	err = models.UpdateJob(job)
 	if err != nil {
 		log.Error("UpdateJob failed:", err)
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 86f111641..9225433cf 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -1011,6 +1011,10 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 {
 						task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 					}
+					if task.EndTime == 0 && task.Status == string(models.ModelArtsStopped) {
+						task.EndTime = timeutil.TimeStampNow()
+						task.Duration = task.ComputeDuration()
+					}
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)

From f8d2c82f3f06eef4477db26e7f2e63fa6b252e95 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 10:29:23 +0800
Subject: [PATCH 03/13] #1654 update

---
 models/cloudbrain.go             | 4 ++++
 routers/api/v1/repo/modelarts.go | 4 +++-
 routers/repo/cloudbrain.go       | 5 ++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index a7a0cbd98..4ee0b7b33 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -165,6 +165,10 @@ func (task *Cloudbrain) ComputeDuration() int64 {
 	return d
 }
 
+func IsTrainJobTerminal(status string) bool {
+	return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled)
+}
+
 type CloudbrainInfo struct {
 	Cloudbrain `xorm:"extends"`
 	User       `xorm:"extends"`
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index cc311c446..4d0e94160 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -140,7 +140,9 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-
+	if job.StartTime == 0 {
+		job.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
+	}
 	job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
 	job.Duration = result.Duration
 	job.TrainJobDuration = result.TrainJobDuration
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 9225433cf..dae8506f0 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -1034,7 +1034,7 @@ func SyncCloudbrainStatus() {
 					task.TrainJobDuration = result.TrainJobDuration
 
 					if task.StartTime == 0 {
-						task.StartTime = timeutil.TimeStamp(result.StartTime)
+						task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 					}
 					if result.Duration != 0 {
 						task.TrainJobDuration = util.AddZero(result.Duration/3600000) + ":" + util.AddZero(result.Duration%3600000/60000) + ":" + util.AddZero(result.Duration%60000/1000)
@@ -1042,6 +1042,9 @@ func SyncCloudbrainStatus() {
 					} else {
 						task.TrainJobDuration = "00:00:00"
 					}
+					if models.IsTrainJobTerminal(task.Status) {
+						task.EndTime = task.StartTime.Add(task.Duration / 1000)
+					}
 
 					err = models.UpdateJob(task)
 					if err != nil {

From 39b5f6291ca5a563a768d610cf168fb2647f03a1 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 10:33:46 +0800
Subject: [PATCH 04/13] #1654 update

---
 routers/api/v1/repo/modelarts.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 4d0e94160..618f6c48d 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -154,6 +154,10 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		job.TrainJobDuration = "00:00:00"
 	}
 
+	if models.IsTrainJobTerminal(job.Status) {
+		job.EndTime = job.StartTime.Add(job.Duration / 1000)
+	}
+
 	err = models.UpdateTrainJobVersion(job)
 	if err != nil {
 		log.Error("UpdateJob failed:", err)

From 594b059a73e785e6bb2464f1935f39071a59acae Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 10:43:41 +0800
Subject: [PATCH 05/13] #1654 update

---
 models/cloudbrain.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 4ee0b7b33..11368f076 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -1395,7 +1395,7 @@ func UpdateTrainJobVersion(job *Cloudbrain) error {
 func updateJobTrainVersion(e Engine, job *Cloudbrain) error {
 	var sess *xorm.Session
 	sess = e.Where("job_id = ? AND version_name=?", job.JobID, job.VersionName)
-	_, err := sess.Cols("status", "train_job_duration").Update(job)
+	_, err := sess.Cols("status", "train_job_duration", "start_time", "end_time").Update(job)
 	return err
 }
 

From 796af34cc1cd669bc76288c74f0aaa87d12eb436 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 11:09:24 +0800
Subject: [PATCH 06/13] #1654 add train job endtime

---
 models/cloudbrain.go             |  6 +++---
 routers/api/v1/repo/modelarts.go | 15 ++++++++++-----
 routers/repo/cloudbrain.go       |  4 ++--
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 11368f076..87a2ebd2c 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -102,7 +102,7 @@ type Cloudbrain struct {
 	ContainerIp          string
 	CreatedUnix          timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix          timeutil.TimeStamp `xorm:"INDEX updated"`
-	Duration             int64
+	Duration             int64              //运行时长 单位秒
 	TrainJobDuration     string
 	Image                string    //镜像名称
 	GpuQueue             string    //GPU类型即GPU队列
@@ -1395,7 +1395,7 @@ func UpdateTrainJobVersion(job *Cloudbrain) error {
 func updateJobTrainVersion(e Engine, job *Cloudbrain) error {
 	var sess *xorm.Session
 	sess = e.Where("job_id = ? AND version_name=?", job.JobID, job.VersionName)
-	_, err := sess.Cols("status", "train_job_duration", "start_time", "end_time").Update(job)
+	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time").Update(job)
 	return err
 }
 
@@ -1475,7 +1475,7 @@ func UpdateInferenceJob(job *Cloudbrain) error {
 func updateInferenceJob(e Engine, job *Cloudbrain) error {
 	var sess *xorm.Session
 	sess = e.Where("job_id = ?", job.JobID)
-	_, err := sess.Cols("status", "train_job_duration").Update(job)
+	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time").Update(job)
 	return err
 }
 func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) {
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 618f6c48d..28756699c 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -144,11 +144,11 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		job.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 	}
 	job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
-	job.Duration = result.Duration
+	job.Duration = result.Duration / 1000
 	job.TrainJobDuration = result.TrainJobDuration
 
 	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(result.Duration/3600000) + ":" + util.AddZero(result.Duration%3600000/60000) + ":" + util.AddZero(result.Duration%60000/1000)
+		job.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
 
 	} else {
 		job.TrainJobDuration = "00:00:00"
@@ -379,17 +379,22 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-
+	if job.StartTime == 0 {
+		job.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
+	}
 	job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
-	job.Duration = result.Duration
+	job.Duration = result.Duration / 1000
 	job.TrainJobDuration = result.TrainJobDuration
 
 	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(result.Duration/3600000) + ":" + util.AddZero(result.Duration%3600000/60000) + ":" + util.AddZero(result.Duration%60000/1000)
+		job.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
 
 	} else {
 		job.TrainJobDuration = "00:00:00"
 	}
+	if models.IsTrainJobTerminal(job.Status) {
+		job.EndTime = job.StartTime.Add(job.Duration / 1000)
+	}
 
 	err = models.UpdateInferenceJob(job)
 	if err != nil {
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index dae8506f0..7f907dd52 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -1030,14 +1030,14 @@ func SyncCloudbrainStatus() {
 
 				if result != nil {
 					task.Status = modelarts.TransTrainJobStatus(result.IntStatus)
-					task.Duration = result.Duration
+					task.Duration = result.Duration / 1000
 					task.TrainJobDuration = result.TrainJobDuration
 
 					if task.StartTime == 0 {
 						task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 					}
 					if result.Duration != 0 {
-						task.TrainJobDuration = util.AddZero(result.Duration/3600000) + ":" + util.AddZero(result.Duration%3600000/60000) + ":" + util.AddZero(result.Duration%60000/1000)
+						task.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
 
 					} else {
 						task.TrainJobDuration = "00:00:00"

From 8ad955372d71298467ddb652b0f065273030e727 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 14:16:32 +0800
Subject: [PATCH 07/13] #1654 fix bug

---
 routers/api/v1/repo/modelarts.go | 4 ++--
 routers/repo/cloudbrain.go       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 28756699c..5476f7bf2 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -155,7 +155,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 	}
 
 	if models.IsTrainJobTerminal(job.Status) {
-		job.EndTime = job.StartTime.Add(job.Duration / 1000)
+		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 
 	err = models.UpdateTrainJobVersion(job)
@@ -393,7 +393,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 		job.TrainJobDuration = "00:00:00"
 	}
 	if models.IsTrainJobTerminal(job.Status) {
-		job.EndTime = job.StartTime.Add(job.Duration / 1000)
+		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 
 	err = models.UpdateInferenceJob(job)
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 7f907dd52..39780f8fb 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -1043,7 +1043,7 @@ func SyncCloudbrainStatus() {
 						task.TrainJobDuration = "00:00:00"
 					}
 					if models.IsTrainJobTerminal(task.Status) {
-						task.EndTime = task.StartTime.Add(task.Duration / 1000)
+						task.EndTime = task.StartTime.Add(task.Duration)
 					}
 
 					err = models.UpdateJob(task)

From 95329d8f074adfc8c64dccc48e4f33f0ac3b3e0f Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 15:06:45 +0800
Subject: [PATCH 08/13] #1654 fix bug

---
 routers/api/v1/repo/modelarts.go | 4 ++--
 routers/repo/cloudbrain.go       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 5476f7bf2..5675ed20a 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -148,7 +148,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 	job.TrainJobDuration = result.TrainJobDuration
 
 	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
+		job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60)
 
 	} else {
 		job.TrainJobDuration = "00:00:00"
@@ -387,7 +387,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 	job.TrainJobDuration = result.TrainJobDuration
 
 	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
+		job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60)
 
 	} else {
 		job.TrainJobDuration = "00:00:00"
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 39780f8fb..cdaa8b515 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -1037,7 +1037,7 @@ func SyncCloudbrainStatus() {
 						task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 					}
 					if result.Duration != 0 {
-						task.TrainJobDuration = util.AddZero(result.Duration/3600) + ":" + util.AddZero(result.Duration%3600/60) + ":" + util.AddZero(result.Duration%60)
+						task.TrainJobDuration = util.AddZero(task.Duration/3600) + ":" + util.AddZero(task.Duration%3600/60) + ":" + util.AddZero(task.Duration%60)
 
 					} else {
 						task.TrainJobDuration = "00:00:00"

From e0b49c1a6319e92cf4fda635651b981259dee50b Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 16:12:48 +0800
Subject: [PATCH 09/13] #1654 fix bug

---
 routers/api/v1/repo/cloudbrain.go | 2 +-
 routers/api/v1/repo/modelarts.go  | 6 +++---
 routers/repo/cloudbrain.go        | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 53a3d9545..48013de84 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -79,7 +79,7 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 		job.ContainerID = taskRes.TaskStatuses[0].ContainerID
 		job.Status = taskRes.TaskStatuses[0].State
 
-		if job.StartTime == 0 {
+		if job.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() {
 			job.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
 		}
 	}
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 5675ed20a..c6e2892ea 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -68,7 +68,7 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-	if job.StartTime == 0 {
+	if job.StartTime == 0 && result.Lease.CreateTime > 0 {
 		job.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 	}
 	job.Status = result.Status
@@ -140,7 +140,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-	if job.StartTime == 0 {
+	if job.StartTime == 0 && result.StartTime > 0 {
 		job.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 	}
 	job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
@@ -379,7 +379,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 		ctx.NotFound(err)
 		return
 	}
-	if job.StartTime == 0 {
+	if job.StartTime == 0 && result.StartTime > 0 {
 		job.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 	}
 	job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index cdaa8b515..9ff299ba9 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -381,7 +381,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) {
 			task.Status = taskRes.TaskStatuses[0].State
 			task.ContainerID = taskRes.TaskStatuses[0].ContainerID
 			task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
-			if task.StartTime == 0 {
+			if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() {
 				task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
 			}
 			err = models.UpdateJob(task)
@@ -961,7 +961,7 @@ func SyncCloudbrainStatus() {
 				task.Status = taskRes.TaskStatuses[0].State
 				if task.Status != string(models.JobWaiting) {
 					task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix()
-					if task.StartTime == 0 {
+					if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() {
 						task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
 					}
 					//if task.Status == string(models.JobFailed) {
@@ -1008,7 +1008,7 @@ func SyncCloudbrainStatus() {
 
 				if result != nil {
 					task.Status = result.Status
-					if task.StartTime == 0 {
+					if task.StartTime == 0 && result.Lease.CreateTime > 0 {
 						task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 					}
 					if task.EndTime == 0 && task.Status == string(models.ModelArtsStopped) {
@@ -1033,7 +1033,7 @@ func SyncCloudbrainStatus() {
 					task.Duration = result.Duration / 1000
 					task.TrainJobDuration = result.TrainJobDuration
 
-					if task.StartTime == 0 {
+					if task.StartTime == 0 && result.StartTime > 0 {
 						task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 					}
 					if result.Duration != 0 {

From 996d386eddf1e23fc205da075dd61893600d1869 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 17:20:21 +0800
Subject: [PATCH 10/13] #1654 fix bug

---
 models/cloudbrain.go              |  8 ++++++++
 routers/api/v1/repo/cloudbrain.go |  4 ++++
 routers/api/v1/repo/modelarts.go  |  6 +++---
 routers/repo/cloudbrain.go        | 30 ++++++++++++++++++------------
 4 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 87a2ebd2c..759732aec 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -169,6 +169,14 @@ func IsTrainJobTerminal(status string) bool {
 	return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled)
 }
 
+func IsModelArtsDebugJobTerminal(status string) bool {
+	return status == string(ModelArtsStopped)
+}
+
+func IsCloudBrainOneDebugJobTerminal(status string) bool {
+	return status == string(JobStopped) || status == string(JobFailed)
+}
+
 type CloudbrainInfo struct {
 	Cloudbrain `xorm:"extends"`
 	User       `xorm:"extends"`
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 48013de84..fcee51b1e 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -85,6 +85,10 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 	}
 
 	if result.JobStatus.State != string(models.JobWaiting) {
+		if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) {
+			job.EndTime = timeutil.TimeStampNow()
+			job.Duration = job.ComputeDuration()
+		}
 		err = models.UpdateJob(job)
 		if err != nil {
 			log.Error("UpdateJob failed:", err)
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index c6e2892ea..402868bb0 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -72,7 +72,7 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 		job.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 	}
 	job.Status = result.Status
-	if job.EndTime == 0 && job.Status == string(models.ModelArtsStopped) {
+	if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) {
 		job.EndTime = timeutil.TimeStampNow()
 		job.Duration = job.ComputeDuration()
 	}
@@ -154,7 +154,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		job.TrainJobDuration = "00:00:00"
 	}
 
-	if models.IsTrainJobTerminal(job.Status) {
+	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 
@@ -392,7 +392,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 	} else {
 		job.TrainJobDuration = "00:00:00"
 	}
-	if models.IsTrainJobTerminal(job.Status) {
+	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
 
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 9ff299ba9..864eb8cb6 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -493,8 +493,10 @@ func CloudBrainStop(ctx *context.Context) {
 		}
 
 		task.Status = string(models.JobStopped)
-		task.EndTime = timeutil.TimeStampNow()
-		task.Duration = task.ComputeDuration()
+		if task.EndTime == 0 {
+			task.EndTime = timeutil.TimeStampNow()
+			task.Duration = task.ComputeDuration()
+		}
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
@@ -588,8 +590,10 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
 		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
 	} else {
 		taskInfo.Status = string(models.JobStopped)
-		taskInfo.EndTime = timeutil.TimeStampNow()
-		taskInfo.Duration = taskInfo.ComputeDuration()
+		if taskInfo.EndTime == 0 {
+			taskInfo.EndTime = timeutil.TimeStampNow()
+			taskInfo.Duration = taskInfo.ComputeDuration()
+		}
 		err = models.UpdateJob(taskInfo)
 		if err != nil {
 			log.Warn("UpdateJob failed", err)
@@ -964,10 +968,10 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 && !taskRes.TaskStatuses[0].StartAt.IsZero() {
 						task.StartTime = timeutil.TimeStamp(taskRes.TaskStatuses[0].StartAt.Unix())
 					}
-					//if task.Status == string(models.JobFailed) {
-					//	task.EndTime = timeutil.TimeStampNow()
-					//	task.Duration = task.ComputeDuration()
-					//}
+					if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) {
+						task.EndTime = timeutil.TimeStampNow()
+						task.Duration = task.ComputeDuration()
+					}
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -988,8 +992,10 @@ func SyncCloudbrainStatus() {
 							continue
 						}
 						task.Status = string(models.JobStopped)
-						task.EndTime = timeutil.TimeStampNow()
-						task.Duration = task.ComputeDuration()
+						if task.EndTime == 0 {
+							task.EndTime = timeutil.TimeStampNow()
+							task.Duration = task.ComputeDuration()
+						}
 						err = models.UpdateJob(task)
 						if err != nil {
 							log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1011,7 +1017,7 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 && result.Lease.CreateTime > 0 {
 						task.StartTime = timeutil.TimeStamp(result.Lease.CreateTime / 1000)
 					}
-					if task.EndTime == 0 && task.Status == string(models.ModelArtsStopped) {
+					if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 						task.EndTime = timeutil.TimeStampNow()
 						task.Duration = task.ComputeDuration()
 					}
@@ -1042,7 +1048,7 @@ func SyncCloudbrainStatus() {
 					} else {
 						task.TrainJobDuration = "00:00:00"
 					}
-					if models.IsTrainJobTerminal(task.Status) {
+					if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
 						task.EndTime = task.StartTime.Add(task.Duration)
 					}
 

From bc2161bd84b2aeaf00b760454b4800ca348d64d0 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Wed, 23 Mar 2022 18:00:37 +0800
Subject: [PATCH 11/13] #1654 update

---
 routers/repo/modelarts.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 6b5eeffbf..c2043fb49 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -409,7 +409,7 @@ func NotebookManage(ctx *context.Context) {
 		}
 
 		task.Status = res.Status
-		if task.Status == string(models.ModelArtsStopped) {
+		if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 			task.EndTime = timeutil.TimeStampNow()
 			task.Duration = task.ComputeDuration()
 		}

From 330df70dbea11beb26585a47251e0dc15d64bf7b Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Thu, 24 Mar 2022 14:17:01 +0800
Subject: [PATCH 12/13] #1654 update

---
 models/cloudbrain.go              | 44 ++++++++++++++++++++++++++-------------
 routers/api/v1/repo/cloudbrain.go |  2 +-
 routers/api/v1/repo/modelarts.go  | 17 +++------------
 routers/repo/cloudbrain.go        | 32 +++++++++++++---------------
 routers/repo/modelarts.go         |  2 +-
 5 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 759732aec..ea6d0338e 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -1,6 +1,7 @@
 package models
 
 import (
+	"code.gitea.io/gitea/modules/util"
 	"encoding/json"
 	"fmt"
 	"strconv"
@@ -102,15 +103,15 @@ type Cloudbrain struct {
 	ContainerIp          string
 	CreatedUnix          timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix          timeutil.TimeStamp `xorm:"INDEX updated"`
-	Duration             int64              //运行时长 单位秒
-	TrainJobDuration     string
-	Image                string    //镜像名称
-	GpuQueue             string    //GPU类型即GPU队列
-	ResourceSpecId       int       //GPU规格id
-	DeletedAt            time.Time `xorm:"deleted"`
-	CanDebug             bool      `xorm:"-"`
-	CanDel               bool      `xorm:"-"`
-	CanModify            bool      `xorm:"-"`
+	Duration             int64              `xorm:"DEFAULT 0"` //运行时长 单位秒
+	TrainJobDuration     string             `xorm:"DEFAULT '00:00:00'"`
+	Image                string             //镜像名称
+	GpuQueue             string             //GPU类型即GPU队列
+	ResourceSpecId       int                //GPU规格id
+	DeletedAt            time.Time          `xorm:"deleted"`
+	CanDebug             bool               `xorm:"-"`
+	CanDel               bool               `xorm:"-"`
+	CanModify            bool               `xorm:"-"`
 	Type                 int
 	BenchmarkTypeID      int
 	BenchmarkChildTypeID int
@@ -154,15 +155,28 @@ type Cloudbrain struct {
 	EndTime               timeutil.TimeStamp
 }
 
-func (task *Cloudbrain) ComputeDuration() int64 {
-	if task.StartTime == 0 || task.EndTime == 0 {
-		return 0
+func (task *Cloudbrain) ComputeAndSetDuration() {
+	var d int64
+	if task.StartTime == 0 {
+		d = 0
+	} else if task.EndTime == 0 {
+		d = time.Now().Unix() - task.StartTime.AsTime().Unix()
+	} else {
+		d = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
 	}
-	d := task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
+
 	if d < 0 {
 		d = 0
 	}
-	return d
+	task.Duration = d
+	task.TrainJobDuration = ConvertDurationToStr(d)
+}
+
+func ConvertDurationToStr(duration int64) string {
+	if duration == 0 {
+		return "00:00:00"
+	}
+	return util.AddZero(duration/3600) + ":" + util.AddZero(duration%3600/60) + ":" + util.AddZero(duration%60)
 }
 
 func IsTrainJobTerminal(status string) bool {
@@ -174,7 +188,7 @@ func IsModelArtsDebugJobTerminal(status string) bool {
 }
 
 func IsCloudBrainOneDebugJobTerminal(status string) bool {
-	return status == string(JobStopped) || status == string(JobFailed)
+	return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded)
 }
 
 type CloudbrainInfo struct {
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index fcee51b1e..b2f529dfb 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -87,8 +87,8 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 	if result.JobStatus.State != string(models.JobWaiting) {
 		if job.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(job.Status) {
 			job.EndTime = timeutil.TimeStampNow()
-			job.Duration = job.ComputeDuration()
 		}
+		job.ComputeAndSetDuration()
 		err = models.UpdateJob(job)
 		if err != nil {
 			log.Error("UpdateJob failed:", err)
diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go
index 402868bb0..d7d011e07 100755
--- a/routers/api/v1/repo/modelarts.go
+++ b/routers/api/v1/repo/modelarts.go
@@ -11,8 +11,6 @@ import (
 	"strconv"
 	"strings"
 
-	"code.gitea.io/gitea/modules/util"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
@@ -74,8 +72,8 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 	job.Status = result.Status
 	if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) {
 		job.EndTime = timeutil.TimeStampNow()
-		job.Duration = job.ComputeDuration()
 	}
+	job.ComputeAndSetDuration()
 	err = models.UpdateJob(job)
 	if err != nil {
 		log.Error("UpdateJob failed:", err)
@@ -147,12 +145,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 	job.Duration = result.Duration / 1000
 	job.TrainJobDuration = result.TrainJobDuration
 
-	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60)
-
-	} else {
-		job.TrainJobDuration = "00:00:00"
-	}
+	job.TrainJobDuration = models.ConvertDurationToStr(job.Duration)
 
 	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
@@ -386,12 +379,8 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 	job.Duration = result.Duration / 1000
 	job.TrainJobDuration = result.TrainJobDuration
 
-	if result.Duration != 0 {
-		job.TrainJobDuration = util.AddZero(job.Duration/3600) + ":" + util.AddZero(job.Duration%3600/60) + ":" + util.AddZero(job.Duration%60)
+	job.TrainJobDuration = models.ConvertDurationToStr(result.Duration)
 
-	} else {
-		job.TrainJobDuration = "00:00:00"
-	}
 	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index f7e36c1ac..85ce02c75 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -488,8 +488,8 @@ func CloudBrainStop(ctx *context.Context) {
 		task.Status = string(models.JobStopped)
 		if task.EndTime == 0 {
 			task.EndTime = timeutil.TimeStampNow()
-			task.Duration = task.ComputeDuration()
 		}
+		task.ComputeAndSetDuration()
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
@@ -585,8 +585,8 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
 		taskInfo.Status = string(models.JobStopped)
 		if taskInfo.EndTime == 0 {
 			taskInfo.EndTime = timeutil.TimeStampNow()
-			taskInfo.Duration = taskInfo.ComputeDuration()
 		}
+		taskInfo.ComputeAndSetDuration()
 		err = models.UpdateJob(taskInfo)
 		if err != nil {
 			log.Warn("UpdateJob failed", err)
@@ -963,8 +963,8 @@ func SyncCloudbrainStatus() {
 					}
 					if task.EndTime == 0 && models.IsCloudBrainOneDebugJobTerminal(task.Status) {
 						task.EndTime = timeutil.TimeStampNow()
-						task.Duration = task.ComputeDuration()
 					}
+					task.ComputeAndSetDuration()
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -987,8 +987,8 @@ func SyncCloudbrainStatus() {
 						task.Status = string(models.JobStopped)
 						if task.EndTime == 0 {
 							task.EndTime = timeutil.TimeStampNow()
-							task.Duration = task.ComputeDuration()
 						}
+						task.ComputeAndSetDuration()
 						err = models.UpdateJob(task)
 						if err != nil {
 							log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1012,8 +1012,8 @@ func SyncCloudbrainStatus() {
 					}
 					if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 						task.EndTime = timeutil.TimeStampNow()
-						task.Duration = task.ComputeDuration()
 					}
+					task.ComputeAndSetDuration()
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1035,12 +1035,7 @@ func SyncCloudbrainStatus() {
 					if task.StartTime == 0 && result.StartTime > 0 {
 						task.StartTime = timeutil.TimeStamp(result.StartTime / 1000)
 					}
-					if result.Duration != 0 {
-						task.TrainJobDuration = util.AddZero(task.Duration/3600) + ":" + util.AddZero(task.Duration%3600/60) + ":" + util.AddZero(task.Duration%60)
-
-					} else {
-						task.TrainJobDuration = "00:00:00"
-					}
+					task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
 					if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
 						task.EndTime = task.StartTime.Add(task.Duration)
 					}
@@ -1090,13 +1085,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
 	for i, task := range ciTasks {
 		ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
 		ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
-		var duration int64
-		if task.Status == string(models.JobRunning) {
-			duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
-		} else {
-			duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
+		if ciTasks[i].TrainJobDuration == "" {
+			var duration int64
+			if task.Status == string(models.JobRunning) {
+				duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
+			} else {
+				duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
+			}
+			ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(duration)
 		}
-		ciTasks[i].TrainJobDuration = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000)
+
 		ciTasks[i].BenchmarkTypeName = ""
 		if task.BenchmarkTypeID > 0 {
 			for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index c2043fb49..690a878e9 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -411,8 +411,8 @@ func NotebookManage(ctx *context.Context) {
 		task.Status = res.Status
 		if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 			task.EndTime = timeutil.TimeStampNow()
-			task.Duration = task.ComputeDuration()
 		}
+		task.ComputeAndSetDuration()
 		err = models.UpdateJob(task)
 		if err != nil {
 			log.Error("UpdateJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"])

From 2a325ff93f97fef0c2e953add821924f7e1e6179 Mon Sep 17 00:00:00 2001
From: chenyifan01 <gitea@fake.local>
Date: Thu, 24 Mar 2022 14:58:05 +0800
Subject: [PATCH 13/13] #1654 update

---
 routers/repo/cloudbrain.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go
index 85ce02c75..d444ea73f 100755
--- a/routers/repo/cloudbrain.go
+++ b/routers/repo/cloudbrain.go
@@ -402,12 +402,6 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) {
 		task.User = user
 	}
 
-	var duration int64
-	if task.Status == string(models.JobRunning) {
-		duration = time.Now().Unix() - int64(task.CreatedUnix)
-	} else {
-		duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
-	}
 	if task.BenchmarkTypeID > 0 {
 		for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
 			if task.BenchmarkTypeID == benchmarkType.Id {
@@ -422,8 +416,16 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName) {
 			}
 		}
 	}
-
-	ctx.Data["duration"] = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000)
+	if task.TrainJobDuration == "" {
+		var duration int64
+		if task.Status == string(models.JobRunning) {
+			duration = time.Now().Unix() - int64(task.CreatedUnix)
+		} else {
+			duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
+		}
+		task.TrainJobDuration = models.ConvertDurationToStr(duration)
+	}
+	ctx.Data["duration"] = task.TrainJobDuration
 	ctx.Data["task"] = task
 	ctx.Data["jobName"] = task.JobName
 	ctx.Data["displayJobName"] = task.DisplayJobName