Merge branch 'V20220415' into zouap

models/cloudbrain.go

@@ -112,7 +112,7 @@ type Cloudbrain struct {
 	SubTaskName          string
 	ContainerID          string
 	ContainerIp          string
-	CreatedUnix          timeutil.TimeStamp `xorm:"INDEX created"`
+	CreatedUnix          timeutil.TimeStamp `xorm:"INDEX"`
 	UpdatedUnix          timeutil.TimeStamp `xorm:"INDEX updated"`
 	Duration             int64              `xorm:"DEFAULT 0"` //运行时长 单位秒
 	TrainJobDuration     string             `xorm:"DEFAULT '00:00:00'"`
@@ -185,6 +185,12 @@ func (task *Cloudbrain) ComputeAndSetDuration() {
 	task.TrainJobDuration = ConvertDurationToStr(d)
 }
 
+func (task *Cloudbrain) CorrectCreateUnix() {
+	if task.StartTime > 0 && task.CreatedUnix > task.StartTime {
+		task.CreatedUnix = task.StartTime
+	}
+}
+
 func (task *Cloudbrain) IsTerminal() bool {
 	status := task.Status
 	return status == string(ModelArtsTrainJobCompleted) || status == string(ModelArtsTrainJobFailed) || status == string(ModelArtsTrainJobKilled) || status == string(ModelArtsStopped) || status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded)
@@ -219,6 +225,7 @@ func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbr
 			task.EndTime = timeutil.TimeStamp(result.JobStatus.CompletedTime / 1000)
 		}
 	}
+	task.CorrectCreateUnix()
 	task.ComputeAndSetDuration()
 }
 
@@ -1465,7 +1472,7 @@ func UpdateTrainJobVersion(job *Cloudbrain) error {
 func updateJobTrainVersion(e Engine, job *Cloudbrain) error {
 	var sess *xorm.Session
 	sess = e.Where("job_id = ? AND version_name=?", job.JobID, job.VersionName)
-	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time").Update(job)
+	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time", "created_unix").Update(job)
 	return err
 }
 
@@ -1554,7 +1561,7 @@ func UpdateInferenceJob(job *Cloudbrain) error {
 func updateInferenceJob(e Engine, job *Cloudbrain) error {
 	var sess *xorm.Session
 	sess = e.Where("job_id = ?", job.JobID)
-	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time").Update(job)
+	_, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time", "created_unix").Update(job)
 	return err
 }
 func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) {

routers/api/v1/repo/modelarts.go

@@ -74,6 +74,7 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {
 	if job.EndTime == 0 && models.IsModelArtsDebugJobTerminal(job.Status) {
 		job.EndTime = timeutil.TimeStampNow()
 	}
+	job.CorrectCreateUnix()
 	job.ComputeAndSetDuration()
 	err = models.UpdateJob(job)
 	if err != nil {
@@ -160,6 +161,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		}
 
 		if result.JobStatus.State != string(models.JobWaiting) {
+			models.ParseAndSetDurationFromCloudBrainOne(result, job)
 			err = models.UpdateJob(job)
 			if err != nil {
 				log.Error("UpdateJob failed:", err)
@@ -177,14 +179,12 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		}
 		job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
 		job.Duration = result.Duration / 1000
-		job.TrainJobDuration = result.TrainJobDuration
-
 		job.TrainJobDuration = models.ConvertDurationToStr(job.Duration)
 
 		if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 			job.EndTime = job.StartTime.Add(job.Duration)
 		}
-
+		job.CorrectCreateUnix()
 		err = models.UpdateTrainJobVersion(job)
 		if err != nil {
 			log.Error("UpdateJob failed:", err)
@@ -417,7 +417,7 @@ func GetModelArtsInferenceJob(ctx *context.APIContext) {
 	if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
 		job.EndTime = job.StartTime.Add(job.Duration)
 	}
-
+	job.CorrectCreateUnix()
 	err = models.UpdateInferenceJob(job)
 	if err != nil {
 		log.Error("UpdateJob failed:", err)

routers/repo/cloudbrain.go

@@ -1170,6 +1170,7 @@ func SyncCloudbrainStatus() {
 					if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) {
 						task.EndTime = timeutil.TimeStampNow()
 					}
+					task.CorrectCreateUnix()
 					task.ComputeAndSetDuration()
 					err = models.UpdateJob(task)
 					if err != nil {
@@ -1196,7 +1197,7 @@ func SyncCloudbrainStatus() {
 					if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
 						task.EndTime = task.StartTime.Add(task.Duration)
 					}
-
+					task.CorrectCreateUnix()
 					err = models.UpdateJob(task)
 					if err != nil {
 						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
@@ -1318,6 +1319,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
 						task.StartTime = timeutil.TimeStamp(startTime / 1000)
 						task.EndTime = task.StartTime.Add(duration)
 					}
+					task.CorrectCreateUnix()
 					task.ComputeAndSetDuration()
 					err = models.UpdateJob(task)
 					if err != nil {