OpenI
/
aiforge
Not watched
1
0
Fork 0
Code
Releases 128 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

Merge branch 'V20220110' of git.openi.org.cn:OpenI/aiforge into cb-opt

pull/1278/head
lewis 3 years ago
parent
e65f269459 acb8cf9dc3
commit
3c3a279b6b
5 changed files with 329 additions and 101 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +6
    -4
      models/cloudbrain.go
  2. +81
    -0
      routers/repo/ai_model_manage.go
  3. +3
    -0
      routers/routes/routes.go
  4. +18
    -3
      templates/repo/modelmanage/index.tmpl
  5. +221
    -94
      templates/repo/modelmanage/showinfo.tmpl

+ 6
- 4
models/cloudbrain.go View File

@@ -19,8 +19,8 @@ type JobType string
type ModelArtsJobStatus string

const (
NPUResource = "NPU"
GPUResource = "CPU/GPU"
NPUResource = "NPU"
GPUResource = "CPU/GPU"

JobWaiting CloudbrainStatus = "WAITING"
JobStopped CloudbrainStatus = "STOPPED"
@@ -210,7 +210,7 @@ type CloudbrainsOptions struct {
JobType string
VersionName string
IsLatestVersion string
JobTypeNot bool
JobTypeNot bool
}

type TaskPod struct {
@@ -975,7 +975,9 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
cond = cond.And(
builder.Eq{"Status": "COMPLETED"},
)

cond = cond.And(
builder.Eq{"job_type": "TRAIN"},
)
cloudbrains := make([]*CloudbrainInfo, 0)
if err := sess.Select("job_id,job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC").
Find(&cloudbrains); err != nil {


+ 81
- 0
routers/repo/ai_model_manage.go View File

@@ -474,6 +474,23 @@ func ShowOneVersionOtherModel(ctx *context.Context) {

func ShowModelTemplate(ctx *context.Context) {
ctx.Data["isModelManage"] = true
repoId := ctx.Repo.Repository.ID
Type := -1
_, count, _ := models.QueryModel(&models.AiModelQueryOptions{
ListOptions: models.ListOptions{
Page: 1,
PageSize: 2,
},
RepoID: repoId,
Type: Type,
New: MODEL_LATEST,
})
ctx.Data["MODEL_COUNT"] = count

_, trainCount, _ := models.QueryModelTrainJobList(repoId)
log.Info("query train count=" + fmt.Sprint(trainCount))

ctx.Data["TRAIN_COUNT"] = trainCount
ctx.HTML(200, tplModelManageIndex)
}

@@ -586,3 +603,67 @@ func ModifyModelInfo(ctx *context.Context) {
}

}

func QueryModelListForPredict(ctx *context.Context) {
repoId := ctx.Repo.Repository.ID
modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{
ListOptions: models.ListOptions{
Page: -1,
PageSize: -1,
},
RepoID: repoId,
Type: -1,
New: -1,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}
log.Info("query return count=" + fmt.Sprint(count))

nameList := make([]string, 0)

nameMap := make(map[string][]*models.AiModelManage)
for _, model := range modelResult {
if _, value := nameMap[model.Name]; !value {
models := make([]*models.AiModelManage, 0)
models = append(models, model)
nameMap[model.Name] = models
nameList = append(nameList, model.Name)
} else {
nameMap[model.Name] = append(nameMap[model.Name], model)
}
}

mapInterface := make(map[string]interface{})
mapInterface["nameList"] = nameList
mapInterface["nameMap"] = nameMap
ctx.JSON(http.StatusOK, mapInterface)
}

func QueryModelFileForPredict(ctx *context.Context) {
id := ctx.Query("ID")
model, err := models.QueryModelById(id)
if err != nil {
log.Error("no such model!", err.Error())
ctx.ServerError("no such model:", err)
return
}
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix)
ctx.JSON(http.StatusOK, fileinfos)
}

func QueryOneLevelModelFile(ctx *context.Context) {
id := ctx.Query("ID")
parentDir := ctx.Query("parentDir")
model, err := models.QueryModelById(id)
if err != nil {
log.Error("no such model!", err.Error())
ctx.ServerError("no such model:", err)
return
}
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir)
ctx.JSON(http.StatusOK, fileinfos)
}

+ 3
- 0
routers/routes/routes.go View File

@@ -992,6 +992,9 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/show_model_child_api", repo.ShowOneVersionOtherModel)
m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList)
m.Get("/query_train_job_version", reqRepoCloudBrainReader, repo.QueryTrainJobVersionList)
m.Get("/query_model_for_predict", reqRepoCloudBrainReader, repo.QueryModelListForPredict)
m.Get("/query_modelfile_for_predict", reqRepoCloudBrainReader, repo.QueryModelFileForPredict)
m.Get("/query_onelevel_modelfile", reqRepoCloudBrainReader, repo.QueryOneLevelModelFile)
m.Group("/:ID", func() {
m.Get("", repo.ShowSingleModel)
m.Get("/downloadsingle", repo.DownloadSingleModelFile)


+ 18
- 3
templates/repo/modelmanage/index.tmpl View File

@@ -17,16 +17,30 @@
<div class="repository release dataset-list view">
{{template "repo/header" .}}
<!-- 列表容器 -->
<div class="ui container active loader" id="loadContainer">
<div class="ui container {{if ne $.MODEL_COUNT 0}}active loader {{end}}" id="loadContainer">
{{template "base/alert" .}}
<div class="ui two column stackable grid" style="display: none;">
<div class="ui two column stackable grid">
<div class="column"></div>
<div class="column right aligned">
<!-- -->
<a class="ui button {{if .Permission.CanWrite $.UnitTypeCloudBrain}} green {{else}} disabled {{end}}" onclick="showcreate(this)">{{$.i18n.Tr "repo.model.manage.import_new_model"}}</a>
</div>
</div>

{{if eq $.MODEL_COUNT 0}}
<div class="ui placeholder segment bgtask-none">
<div class="ui icon header bgtask-header-pic"></div>
<div class="bgtask-content-header">未创建过模型</div>
<div class="bgtask-content">
{{if $.RepoIsEmpty}}
<div class="bgtask-content-txt">代码版本：您还没有初始化代码仓库，请先<a href="{{.RepoLink}}">创建代码版本；</a></div>
{{end}}
{{if eq $.TRAIN_COUNT 0}}
<div class="bgtask-content-txt">训练任务：你还没创建过训练任务，请先创建训练任务。</div>
{{end}}
<div class="bgtask-content-txt">使用说明：可以参考启智AI协作平台<a href="https://git.openi.org.cn/zeizei/OpenI_Learning">小白训练营课程。</a></div>
</div>
</div>
{{else}}
<!-- 中下列表展示区 -->
<div class="ui grid" style="display: none;">
<div class="row" style="padding-top: 0;">
@@ -38,6 +52,7 @@
</div>
</div>
</div>
{{end}}

</div>



+ 221
- 94
templates/repo/modelmanage/showinfo.tmpl View File

@@ -78,100 +78,121 @@
</select>
</h4>
<div id="showInfo" style="border:1px solid #e2e2e2;padding: 20px 60px;margin-top:24px">
<div class="half-table">
<span class="model_header_text">基本信息</span>
<table class="tableStyle" style="margin-top:20px;">
<tbody>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.model_name"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="ModelName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.version"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Version" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.migrate_items_labels"}}</td>
<td class="ti-text-form-content">
<div id="Label" style="overflow: hidden;width: 95%;">
<div class="ui pointing secondary menu" style="border-bottom: 1px solid rgba(34,36,38,.15);">
<a class="active item" data-tab="first">{{$.i18n.Tr "repo.modelarts.train_job.config"}}</a>
<a class="item" data-tab="second">{{$.i18n.Tr "repo.model_download"}}</a>
</div>
<div class="ui tab active" data-tab="first">
<div class="half-table">
<span class="model_header_text">基本信息</span>
<table class="tableStyle" style="margin-top:20px;">
<tbody>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.model_name"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="ModelName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.version"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Version" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.migrate_items_labels"}}</td>
<td class="ti-text-form-content">
<div id="Label" style="overflow: hidden;width: 95%;">
</div>
</div>

</td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.model_size"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Size" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.createtime"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="CreateTime" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.description"}}</td>
<td class="ti-text-form-content" >
<div id="edit-td" style="display:flex">
<span id="Description" title="" class="iword-elipsis"></span>
<i id="edit-pencil" data-id="" data-desc="" class="pencil alternate icon" style="cursor:pointer;vertical-align: top;" id="editor" onclick="editorFn(this)"></i>
</div>
</td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.code_version"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="CodeBranch" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.start_file"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="BootFile" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.train_dataset"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="DatasetName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Parameters" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="EngineName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.standard"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="FlavorName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.compute_node"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="WorkServerNumber" title=""></span></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.model_size"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Size" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.createtime"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="CreateTime" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.description"}}</td>
<td class="ti-text-form-content" >
<div id="edit-td" style="display:flex">
<span id="Description" title="" class="iword-elipsis"></span>
<i id="edit-pencil" data-id="" data-desc="" class="pencil alternate icon" style="cursor:pointer;vertical-align: top;" id="editor" onclick="editorFn(this)"></i>
</div>
</td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.code_version"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="CodeBranch" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.start_file"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="BootFile" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.train_dataset"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="DatasetName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Parameters" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="EngineName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.standard"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="FlavorName" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.compute_node"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="WorkServerNumber" title=""></span></td>
</tr>
</tbody>
</table>
</div>
<div class="half-table">
<span class="model_header_text">{{$.i18n.Tr "repo.model.manage.model_accuracy"}}</span>
<table class="tableStyle" style="margin-top:20px;">
<tbody>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Accuracy"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Accuracy" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">F1</td>
<td class="ti-text-form-content word-elipsis"><span id="F1" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Precision"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Precision" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Recall"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Recall" title=""></span></td>
</tr>
</tbody>
</table>
</div>
<div style="clear: both;"></div>
</div>
<div class="half-table">
<span class="model_header_text">{{$.i18n.Tr "repo.model.manage.model_accuracy"}}</span>
<table class="tableStyle" style="margin-top:20px;">
<tbody>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Accuracy"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Accuracy" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">F1</td>
<td class="ti-text-form-content word-elipsis"><span id="F1" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Precision"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Precision" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.model.manage.Recall"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="Recall" title=""></span></td>
</tr>
</tbody>
</table>
<div class="ui tab" data-tab="second">
<input type="hidden" name="model" value="-1">
<input type="hidden" name="modelback" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb'>
<div class="active section"></div>
<div class="divider"> / </div>

</div>
<div id="dir_list">
</div>
</div>
<div style="clear: both;"></div>
</div>
</div>
</div>
@@ -179,6 +200,9 @@
<script>
let url = location.href.split('show_model')[0]
let ID = location.search.split('?name=').pop()
$(document).ready(function(){
$('.secondary.menu .item').tab();
});
$(document).ready(loadInfo);
function changeInfo(version){
$.get(`${url}show_model_info_api?name=${ID}`,(data)=>{
@@ -190,6 +214,7 @@ function changeInfo(version){
let [initObj,initModelAcc,id] = returnArray
editorCancel('','')
renderInfo(initObj,initModelAcc,id)
loadModelFile(versionData[0].ID,versionData[0].Version,'','','init')
})
}
function loadInfo(){
@@ -206,6 +231,7 @@ function loadInfo(){
returnArray = transObj(data)
let [initObj,initModelAcc,id] = returnArray
renderInfo(initObj,initModelAcc,id)
loadModelFile(data[0].ID,data[0].Version,'','','init')
})
}
function transObj(data){
@@ -297,7 +323,6 @@ function renderInfo(obj,accObj,id){
let descriptionText=obj[key].replace(/\r\n|\n/g,'enter;')
$(`#${key}`).text(obj[key])
$(`#${key}`).attr("title",obj[key])

$('#edit-pencil').attr("data-id",id)
$('#edit-pencil').attr("data-desc",descriptionText)
}
@@ -333,8 +358,6 @@ function renderInfo(obj,accObj,id){
$(`#${key}`).text(parameter)
$(`#${key}`).attr("title",parameter)
}

}
else{
$(`#${key}`).text(obj[key])
@@ -348,4 +371,108 @@ function renderInfo(obj,accObj,id){
}
}

function loadModelFile(ID,version_name,parents,filename,init){
$.get(`${url}query_onelevel_modelfile?ID=${ID}&parentDir=${parents}`, (data) => {
$('#dir_list').empty()
renderDir(data,ID,version_name)
if(init==="init"){
$('input[name=model]').val("")
$('input[name=modelback]').val(version_name)
$('#file_breadcrumb').empty()
let htmlBread = ""
htmlBread += `<div class='active section'>${version_name}</div>`
htmlBread += "<div class='divider'> / </div>"
$('#file_breadcrumb').append(htmlBread)
}else{
renderBrend(ID,version_name,parents,filename,init)
}
})
}
function renderSize(value){
if(null==value||value==''){
return "0 Bytes";
}
var unitArr = new Array("Bytes","KB","MB","GB","TB","PB","EB","ZB","YB");
var index=0;
var srcsize = parseFloat(value);
index=Math.floor(Math.log(srcsize)/Math.log(1024));
var size =srcsize/Math.pow(1024,index);
size=size.toFixed(2);//保留的小数位数
return size+unitArr[index];
}

function renderBrend(ID,version_name,parents,filename,init){
if(init=="folder"){
let htmlBrend = ""
let sectionName=$('#file_breadcrumb .active.section').text()
let parents1 = $('input[name=model]').val()
let filename1 = $('input[name=modelback]').val()
if(parents1===""){
$('#file_breadcrumb .active.section').replaceWith(`<a class='section' onclick="loadModelFile('${ID}','${version_name}','${parents1}','','init')">${sectionName}</a>`)
}else{
$('#file_breadcrumb .active.section').replaceWith(`<a class='section' onclick="loadModelFile('${ID}','${version_name}','${parents1}','${filename1}')">${sectionName}</a>`)
}
htmlBrend += `<div class='active section'>${filename}</div>`
htmlBrend += "<div class='divider'> / </div>"
$('#file_breadcrumb').append(htmlBrend)
$('input[name=model]').val(parents)
$('input[name=modelback]').val(filename)
}else{
$('input[name=model]').val(parents)
$('input[name=modelbac]').val(filename)
$('#file_breadcrumb a.section:contains(${filename})').nextAll().remove()
$('#file_breadcrumb a.section:contains(${filename})').replaceWith(`<div class='active section'>${filename}</div>`)
$('#file_breadcrumb div.section:contains(${filename})').append("<div class='divider'> / </div>")
}
}
function renderDir(data,ID,version_name){
let html=""
html += "<div class='ui grid' style='margin:0;'>"
html += "<div class='row' style='padding: 0;'>"
html += "<div class='ui sixteen wide column' style='padding:1rem;'>"
html += "<div class='dir list'>"
html += "<table id='repo-files-table' class='ui single line table pad20'>"
html += '<tbody>'
for(let i=0;i<data.length;i++){
let dirs_size = renderSize(data[i].Size)
html += "<tr>"
html += "<td class='name six wid'>"
html += "<span class='truncate'>"
html += "<span class='octicon octicon-file-directory'>"
html += "</span>"
if(data[i].IsDir){
html += `<a onclick="loadModelFile('${ID}','${version_name}','${data[i].ParenDir}','${data[i].FileName}','folder')">`
html += "<span class='fitted'><i class='folder icon' width='16' height='16' aria-hidden='true'></i>" + data[i].FileName + "</span>"
}else{
html += `<a href="${url}${ID}/downloadsingle?parentDir=${data[i].ParenDir}&fileName=${data[i].FileName}">`
html += "<span class='fitted'><i class='file icon' width='16' height='16' aria-hidden='true'></i>" + data[i].FileName + "</span>"
}
html += '</a>'
html += "</span>"
html += "</td>"
html += "<td class='message seven wide'>"
if(data[i].IsDir){
html += "<span class='truncate has-emoji'></span>"
}else{
html += "<span class='truncate has-emoji'>"+ `${dirs_size}` + "</span>"
}
html += "</td>"

html += "<td class='text right age three wide'>"
html += "<span class='truncate has-emoji'>" + data[i].ModTime + "</span>"
html += "</td>"
html += "</tr>"
}
html += "</tbody>"
html += "</table>"
html += "</div>"
html += "</div>"
html += "</div>"
html += "</div>"
$('#dir_list').append(html)
}
</script>

Write Preview
Loading…
Cancel
Save