diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index b444dd008..9fd00a763 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -385,6 +385,17 @@ CONN_MAX_LIFETIME = 3s ; Database maximum number of open connections, default is 0 meaning no maximum MAX_OPEN_CONNS = 0 +[database_statistic] +DB_TYPE = postgres +HOST = 127.0.0.1:5432 +NAME = statistic +USER = +PASSWD = +SCHEMA = +SSL_MODE = disable +CHARSET = utf8 +PATH = + [indexer] ; Issue indexer type, currently support: bleve, db or elasticsearch, default is bleve ISSUE_INDEXER_TYPE = bleve diff --git a/models/attachment.go b/models/attachment.go index 418d7c881..684a38b21 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -464,3 +464,12 @@ func CanDelAttachment(isSigned bool, user *User, attach *Attachment) bool { } return false } + +func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) { + total, err := x.Where("dataset_id = ?", datasetID).SumInt(&Attachment{}, "size") + if err != nil { + return 0, err + } + + return total, nil +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index c26f8a2b6..4b2bec8e6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -687,7 +687,7 @@ func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { return false } - if user.ID == job.UserID || user.IsAdmin || permission.AccessMode >= AccessModeAdmin { + if (user.ID == job.UserID && permission.AccessMode >= AccessModeWrite) || user.IsAdmin || permission.AccessMode >= AccessModeAdmin { return true } return false diff --git a/models/issue_comment.go b/models/issue_comment.go old mode 100644 new mode 100755 index f7017435d..60d38452c --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -1016,3 +1016,19 @@ func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID }) return err } + +func GetCommentCountByRepoID(repoID int64) (int64, error) { + //sql := fmt.Sprintf("select count(1) from comment where issue_id in (select id from issue where repo_id = %d) and type = %d;", repoID, CommentTypeComment) + //res, err := x.Query(sql) + //if err != nil { + // return 0, err + //} + //return int64(binary.BigEndian.Uint64(res[0]["count"])), nil + + total, err := x.Where("issue_id in (select id from issue where repo_id = ?) and type = ?", repoID, CommentTypeComment).Count(&Comment{}) + if err != nil { + return 0, err + } + + return total, nil +} diff --git a/models/models.go b/models/models.go index 3927dfbf0..412148235 100755 --- a/models/models.go +++ b/models/models.go @@ -136,7 +136,7 @@ func init() { ) tablesStatistic = append(tablesStatistic, - new(FileChunk), + new(RepoStatistic), new(UserBusinessAnalysis), ) diff --git a/models/release.go b/models/release.go old mode 100644 new mode 100755 diff --git a/models/repo_statistic.go b/models/repo_statistic.go new file mode 100755 index 000000000..b987f4f46 --- /dev/null +++ b/models/repo_statistic.go @@ -0,0 +1,60 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "fmt" +) + +// RepoStatistic statistic info of all repository +type RepoStatistic struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"unique(s) NOT NULL"` + Date string `xorm:"unique(s) NOT NULL"` + NumWatches int64 `xorm:"NOT NULL DEFAULT 0"` + NumStars int64 `xorm:"NOT NULL DEFAULT 0"` + NumForks int64 `xorm:"NOT NULL DEFAULT 0"` + NumDownloads int64 `xorm:"NOT NULL DEFAULT 0"` + NumComments int64 `xorm:"NOT NULL DEFAULT 0"` + NumVisits int64 `xorm:"NOT NULL DEFAULT 0"` + NumClosedIssues int64 `xorm:"NOT NULL DEFAULT 0"` + NumVersions int64 `xorm:"NOT NULL DEFAULT 0"` + //develop months + NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0"` + NumCommits int64 `xorm:"NOT NULL DEFAULT 0"` + NumIssues int64 `xorm:"NOT NULL DEFAULT 0"` + NumPulls int64 `xorm:"NOT NULL DEFAULT 0"` + IssueFixedRate float32 `xorm:"NOT NULL"` + NumContributor int64 `xorm:"NOT NULL DEFAULT 0"` + NumKeyContributor int64 `xorm:"NOT NULL DEFAULT 0"` + + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func DeleteRepoStatDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&RepoStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func InsertRepoStat(repoStat *RepoStatistic) (int64, error) { + return xStatistic.Insert(repoStat) +} diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a00aef0c8..a7d549cd4 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -55,6 +55,15 @@ type UserBusinessAnalysis struct { //user RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + + //openi index + OpenIIndex int `xorm:"NOT NULL DEFAULT 0"` + //user Email string `xorm:"NOT NULL"` @@ -100,6 +109,7 @@ func CountData(wikiCountMap map[string]int) { } CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) for i, userRecord := range userList { var dateRecord UserBusinessAnalysis @@ -176,6 +186,12 @@ func CountData(wikiCountMap map[string]int) { dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] } + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } + dateRecord.CommitModelCount = 0 statictisSess := xStatistic.NewSession() @@ -322,6 +338,25 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { } +func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + sess.Select("id,owner_id,name").Table("repository").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + resultMap := make(map[int64]int) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + return resultMap + +} + func subMonth(t1, t2 time.Time) (month int) { y1 := t1.Year() y2 := t2.Year() diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go old mode 100644 new mode 100755 index f42710618..26cd16778 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -163,6 +163,28 @@ func registerHandleBlockChainUnSuccessCommits() { }) } +func registerHandleRepoStatistic() { + RegisterTaskFatal("handle_repo_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.RepoStatisticAuto() + return nil + }) +} + +func registerHandleUserStatistic() { + RegisterTaskFatal("handle_user_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.TimingCountData() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -177,4 +199,7 @@ func initBasicTasks() { registerHandleBlockChainUnSuccessRepos() registerHandleBlockChainMergedPulls() registerHandleBlockChainUnSuccessCommits() + + registerHandleRepoStatistic() + registerHandleUserStatistic() } diff --git a/public/self/labelTaskPage.js b/public/self/labelTaskPage.js index 4ce48407a..a160c3961 100644 --- a/public/self/labelTaskPage.js +++ b/public/self/labelTaskPage.js @@ -23,7 +23,7 @@ var pageSize = 10; var tableData; var tablePageData; -var preDictTaskData; +var modelListData; var dataSetTaskData; var userInfoData; @@ -35,8 +35,8 @@ console.log("repoId=" + repoId); function setDataSetTask(){ - dataset_task_list(); - display_createdatasetlabel(0); + //dataset_task_list(); + //display_createdatasetlabel(0); //getUser(); //dislpayUser(); getLabelPropertyTask(); @@ -121,46 +121,31 @@ function countLabel(){ } -function display_createdatasetlabel(sindex=-1){ - var html=""; - for (var i=0;i"+dataSetTaskData[i].task_name+ - ""; - $("#datasetlabeltaskname").attr({value:dataSetTaskData[i].task_name + "-人工标注"}); - }else{ - var row = ""; - } - html=html+row; - } - console.log(html); - document.getElementById('dataset_list').innerHTML=html; -} + function setPredictTask(){ - pre_predict_task_list(); - display_createlabel(0); - getUser(); - dislpayUser(); - getLabelPropertyTask(); - displayLabelPropertyTask(); + //dataset_task_list(); + //display_createdatasetlabel(1); + + + get_model_list(); + + displayModelTask(); + $(".ui.predict.modal").modal("show"); } -function pre_predict_task_list(){ +function get_model_list(){ $.ajax({ type:"GET", - url:ip + "/api/pre-predict-taskforLabel/", + url:ip + "/api/queryAlgModelForAutoLabel/", headers: { authorization:token, }, dataType:"json", async:false, success:function(json){ - preDictTaskData = json; + modelListData = json; console.log(json); // return json.token; }, @@ -170,12 +155,26 @@ function pre_predict_task_list(){ }); } -function sele_Change(sele){ - var predictTaskName = $('#pre_predict_task_for_label option:selected').text(); - console.log("select predictTaskName =" + predictTaskName); - $("#labeltaskname").attr({value:predictTaskName+"-人工标注"}); +function displayModelTask(){ + var html=""; + for (var i=0;i"+modelListData[i].model_name + + ""; + }else{ + var row = ""; + } + html=html+row; + } + console.log(html); + + document.getElementById('model_list').innerHTML=html; } + function sele_export_Change(sele){ var isNeedPicture = $('#isNeedPicture option:selected').val(); @@ -201,27 +200,13 @@ function dataset_sele_Change(sele){ $("#datasetlabeltaskname").attr({value:dataset_listName+"-人工标注"}); } - - -function display_createlabel(sindex=-1){ - var html=""; - for (var i=0;i"+preDictTaskData[i].task_name+ - ""; - $("#labeltaskname").attr({value:preDictTaskData[i].task_name + "-人工标注"}); - }else{ - var row = ""; - } - html=html+row; - } - console.log(html); - document.getElementById('pre_predict_task_for_label').innerHTML=html; +function dataset_auto_sele_Change(sele){ + var dataset_listName = $('#dataset_list_auto option:selected').text(); + console.log("select dataset_list_auto =" + dataset_listName); + $("#autolabeltaskname").attr({value:dataset_listName+"-自动标注"}); } + var createsucced; function submit_datasettask(){ @@ -242,7 +227,7 @@ function submit_datasettask(){ } var labelpropertytaskid = $('#labelpropertytask_dataset option:selected').val(); createsucced = true; - label_task_create(task_name, relate_task_id, 2,assign_user_id,labelpropertytaskid); + label_task_create(task_name, relate_task_id, 2,assign_user_id,labelpropertytaskid,-1); if(createsucced){ $(".ui.dataset.modal").modal("hide"); //$("#labelDataModal").modal('hide'); @@ -250,25 +235,30 @@ function submit_datasettask(){ page(0,pageSize); } -function submit_labeltask(){ - console.log($('#labeltaskname').val()); - var task_name = $('#labeltaskname').val(); +function submit_autolabeltask(){ + console.log($('#autolabeltaskname').val()); + var task_name = $('#autolabeltaskname').val(); if (isEmpty(task_name) || task_name.length > 32){ - alert("人工标注任务名称不能为空或者不能超过32个字符。"); + alert("自动标注任务名称不能为空或者不能超过32个字符。"); return; } - var relate_task_id = $('#pre_predict_task_for_label option:selected').val(); + var model_id = $('#model_list option:selected').val(); + if(isEmpty(model_id)){ + alert("标注模型不能为空。"); + return; + } + var relate_task_id = $('#dataset_list_auto option:selected').val(); if(isEmpty(relate_task_id)){ - alert("关联的自动标注任务不能为空。"); + alert("数据集对象不能为空。"); return; } var assign_user_id = $('#label_assign_user option:selected').val(); if(isEmpty(assign_user_id)){ assign_user_id = token; } - var labelpropertytaskid = $('#labelpropertytask_dataset option:selected').val(); + var labelpropertytaskid = $('#labelpropertytask_auto option:selected').val(); createsucced = true; - label_task_create(task_name, relate_task_id, 1,assign_user_id,labelpropertytaskid); + label_task_create(task_name, relate_task_id, 1,assign_user_id,labelpropertytaskid,model_id); if(createsucced){ $("#labelModal").modal('hide'); } @@ -276,10 +266,10 @@ function submit_labeltask(){ } -function label_task_create(task_name, relate_task_id, taskType,assign_user_id,labelpropertytaskid){ +function label_task_create(task_name, relate_task_id, taskType,assign_user_id,labelpropertytaskid,model_id){ - var task_flow_type = $('#task_flow_type option:selected').val(); + var task_flow_type = $('#task_flow_type option:selected').val(); var relate_other_label_task = []; if(task_flow_type == 2){ @@ -309,20 +299,21 @@ function label_task_create(task_name, relate_task_id, taskType,assign_user_id,la 'assign_user_id':assign_user_id, 'task_flow_type':task_flow_type, 'relate_task_id':relate_task_id,//task id - 'relate_other_label_task': relate_other_label_task_jsonstr, + 'relate_other_label_task': relate_other_label_task_jsonstr, "taskType": taskType, - "appid": repoId, - "createUserName":userName, - "labelPropertyTaskId":labelpropertytaskid + "appid": repoId, + "createUserName":userName, + "labelPropertyTaskId":labelpropertytaskid, + "modelId":model_id }), success:function(res){ console.log(res); if(res.code == 0){ - alert("人工标注任务创建成功!"); + alert("自动标注任务创建成功!"); createsucced = true; } else{ - alert("创建人工标注任务失败，" + res.message); + alert("创建自动标注任务失败，" + res.message); createsucced = false; } }, @@ -433,11 +424,11 @@ function delete_labeltask(){ return; } var Check = $("table[id='label_task_list'] input[type=checkbox]:checked");//在table中找input下类型为checkbox属性为选中状态的数据 -       Check.each(function () {//遍历 -             var row = $(this).parent("td").parent("tr");//获取选中行 -             var id = row.find("[id='labeltask_id']").html();//获取name='Sid'的值 -             delete_labeltask_byid(id); -         }); + Check.each(function () {//遍历 + var row = $(this).parent("td").parent("tr");//获取选中行 + var id = row.find("[id='labeltask_id']").html();//获取name='Sid'的值 + delete_labeltask_byid(id); + }); page(0,pageSize); } @@ -480,13 +471,13 @@ function delete_labeltask_byid(label_task_id){ function getTaskTypeDesc(task_type){ if(task_type == 1){ - return "自动标注结果"; + return "图片-自动标注"; }else if(task_type == 2){ - return "原始数据集-图片"; + return "图片"; }else if(task_type == 3){ - return "原始数据集-CT影像"; + return "CT影像"; }else if(task_type == 4){ - return "原始数据集-视频"; + return "视频"; } return "其它"; } @@ -508,6 +499,12 @@ function getTaskSataus(task_status,task_status_desc){ else if(task_status == -1){ return "关联的数据集已经被删除。" } + else if(task_status == 20){ + return "自动标注进行中:" + task_status_desc; + }else if(task_status == 21){ + return task_status_desc; + } + return ""; } function getVerify(task_status,id,task_type){ @@ -522,7 +519,7 @@ function getVerify(task_status,id,task_type){ } function getLabel(task_status,id,task_type,task_flow_type){ - if(task_status == 0 && (userType == 1 || userType == 0)){ + if((task_status == 0 || task_status == 21) && (userType == 1 || userType == 0)){ return "" + getLabelDesc(task_flow_type) + "标注
"; }else{ return ""; @@ -570,8 +567,59 @@ function display_list(){ $('#label_task_list tr').find('td:eq(1)').hide(); $('#label_task_list tr').find('th:eq(1)').hide(); -} + isNeedToRefresh = false; + taskNeedIntervalToRefresh(); +} + +var isNeedToRefresh; +var refreshTimeId = []; +var refreshCount; +var refreshMaxTime; + +function taskNeedIntervalToRefresh(){ + var isNeedToSetInterval = false; + refreshMaxTime= 1; + if(!isEmpty(tableData)){ + for (var i=0;i
- + {{if .User.Name}} + + {{else}} + + {{end}}
diff --git a/templates/repo/datasets/dataset_list.tmpl b/templates/repo/datasets/dataset_list.tmpl index 47716418a..5cb35aa77 100755 --- a/templates/repo/datasets/dataset_list.tmpl +++ b/templates/repo/datasets/dataset_list.tmpl @@ -5,7 +5,7 @@ {{range .Attachments}}
-
+
{{.Size | FileSize}} {{svg "octicon-cloud-download" 16}} {{.Name}} @@ -13,7 +13,7 @@
- {{svg "octicon-flame" 16}} {{(.DownloadCount | PrettyNumber)}} + {{svg "octicon-flame" 16}} {{(.DownloadCount | PrettyNumber)}} {{svg "octicon-file-binary" 16}}
diff --git a/templates/repo/datasets/label/index.tmpl b/templates/repo/datasets/label/index.tmpl index db65ad251..cb268c5b9 100644 --- a/templates/repo/datasets/label/index.tmpl +++ b/templates/repo/datasets/label/index.tmpl @@ -17,9 +17,11 @@
{{template "repo/header" .}} + +
-

人工标注任务列表

+

标注任务列表

@@ -45,8 +47,8 @@
- - + + @@ -65,7 +67,11 @@
@@ -105,44 +111,50 @@ +
- +