Browse Source

Merge pull request 'block-chain' (#147) from block-chain into develop

Reviewed-by: 史梦园 <1729788216@qq.com>
master
史梦园 4 years ago
parent
commit
3aefcbb92a
13 changed files with 594 additions and 2 deletions
  1. +4
    -0
      custom/conf/app.ini.sample
  2. +11
    -0
      models/action.go
  3. +86
    -0
      models/blockchain.go
  4. +1
    -0
      models/models.go
  5. +25
    -0
      models/repo.go
  6. +26
    -1
      models/user.go
  7. +14
    -0
      modules/blockchain/blockchain.go
  8. +150
    -0
      modules/blockchain/resty.go
  9. +8
    -0
      modules/setting/setting.go
  10. +13
    -0
      modules/timer/timer.go
  11. +1
    -1
      modules/worker/task.go
  12. +250
    -0
      routers/repo/blockchain.go
  13. +5
    -0
      routers/routes/routes.go

+ 4
- 0
custom/conf/app.ini.sample View File

@@ -1065,3 +1065,7 @@ HOST = http://192.168.202.90:3366/
HOST = http://192.168.207.34:39987
USER = cW4cMtH24eoWPE7X
PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC

[blockchain]
HOST = http://192.168.207.84:3002/
COMMIT_VALID_DATE = 2021-01-15

+ 11
- 0
models/action.go View File

@@ -67,6 +67,7 @@ type Action struct {
IsDeleted bool `xorm:"INDEX NOT NULL DEFAULT false"`
RefName string
IsPrivate bool `xorm:"INDEX NOT NULL DEFAULT false"`
IsTransformed bool `xorm:"INDEX NOT NULL DEFAULT false"`
Content string `xorm:"TEXT"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
}
@@ -344,3 +345,13 @@ func GetFeeds(opts GetFeedsOptions) ([]*Action, error) {

return actions, nil
}

func GetUnTransformedActions() ([]*Action, error) {
actions := make([]*Action, 0, 10)
err := x.Where("op_type = ?", ActionCommitRepo).
And("content != ''").
And("is_transformed = ?", false).
And("to_timestamp(created_unix) >= ?", setting.CommitValidDate).
Find(&actions)
return actions, err
}

+ 86
- 0
models/blockchain.go View File

@@ -0,0 +1,86 @@
package models

import (
"code.gitea.io/gitea/modules/timeutil"
"fmt"
"time"
)

type BlockChainCommitStatus int
const (
BlockChainCommitInit BlockChainCommitStatus = iota
BlockChainCommitSuccess
BlockChainCommitFailed
)
type BlockChain struct {
ID int64 `xorm:"pk autoincr"`
CommitID string `xorm:"INDEX NOT NULL"`
Contributor string `xorm:"INDEX NOT NULL"`
ContractAddress string `xorm:"INDEX NOT NULL"`
Status BlockChainCommitStatus `xorm:"INDEX NOT NULL DEFAULT 0"`
Amount int64 `xorm:"INDEX"`
UserID int64 `xorm:"INDEX"`
RepoID int64 `xorm:"INDEX"`
TransactionHash string `xorm:"INDEX"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
DeletedAt time.Time `xorm:"deleted"`

User *User `xorm:"-"`
Repo *Repository `xorm:"-"`
}

func getBlockChainByID(e Engine, id int64) (*BlockChain, error) {
blockChain := new(BlockChain)
has, err := e.ID(id).Get(blockChain)
if err != nil {
return nil, err
} else if !has {
return nil, fmt.Errorf("get block_chain by id failed(%d)", id)
}
return blockChain, nil
}

func GetBlockChainByID(id int64) (*BlockChain, error) {
return getBlockChainByID(x, id)
}

func getBlockChainByCommitID(e Engine, commitID string) (*BlockChain, error) {
blockChain := new(BlockChain)
has, err := e.Where("commit_id = ?", commitID).Get(blockChain)
if err != nil {
return nil, err
} else if !has {
return nil, fmt.Errorf("get block_chain by commitID failed(%s)", commitID)
}
return blockChain, nil
}

func GetBlockChainByCommitID(commitID string) (*BlockChain, error) {
return getBlockChainByCommitID(x, commitID)
}

func updateBlockChainCols(e Engine, blockChain *BlockChain, cols ...string) error {
_, err := e.ID(blockChain.ID).Cols(cols...).Update(blockChain)
return err
}

func UpdateBlockChainCols(blockChain *BlockChain, cols ...string) error {
return updateBlockChainCols(x, blockChain, cols...)
}

func GetBlockChainUnSuccessCommits() ([]*BlockChain, error) {
blockChains := make([]*BlockChain, 0, 10)
return blockChains, x.
Where("status != ?", BlockChainCommitSuccess).
Find(&blockChains)
}

func InsertBlockChain(blockChain *BlockChain) (_ *BlockChain, err error) {

if _, err := x.Insert(blockChain); err != nil {
return nil, err
}

return blockChain, nil
}

+ 1
- 0
models/models.go View File

@@ -128,6 +128,7 @@ func init() {
new(Dataset),
new(Cloudbrain),
new(FileChunk),
new(BlockChain),
)

gonicNames := []string{"SSL", "UID"}


+ 25
- 0
models/repo.go View File

@@ -6,6 +6,7 @@
package models

import (
"code.gitea.io/gitea/modules/blockchain"
"context"
"crypto/md5"
"errors"
@@ -134,6 +135,7 @@ func NewRepoContext() {

// RepositoryStatus defines the status of repository
type RepositoryStatus int
type RepoBlockChainStatus int

// all kinds of RepositoryStatus
const (
@@ -141,6 +143,12 @@ const (
RepositoryBeingMigrated // repository is migrating
)

const (
RepoBlockChainInit RepoBlockChainStatus = iota
RepoBlockChainSuccess
RepoBlockChainFailed
)

// Repository represents a git repository.
type Repository struct {
ID int64 `xorm:"pk autoincr"`
@@ -195,6 +203,11 @@ type Repository struct {
// Avatar: ID(10-20)-md5(32) - must fit into 64 symbols
Avatar string `xorm:"VARCHAR(64)"`

//blockchain
ContractAddress string `xorm:"INDEX"`
Balance int64 `xorm:"NOT NULL DEFAULT 0"`
BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0"`

CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
@@ -1051,6 +1064,11 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository) (err error
return err
}

_, err = blockchain.NewRepo(string(repo.ID), u.PublicKey, repo.Name)
if err != nil {
log.Error("newRepo failed:", err.Error())
}

// insert units for repo
var units = make([]RepoUnit, 0, len(DefaultRepoUnits))
for _, tp := range DefaultRepoUnits {
@@ -2388,3 +2406,10 @@ func updateRepositoryCols(e Engine, repo *Repository, cols ...string) error {
func UpdateRepositoryCols(repo *Repository, cols ...string) error {
return updateRepositoryCols(x, repo, cols...)
}

func GetBlockChainUnSuccessRepos() ([]*Repository, error) {
repos := make([]*Repository, 0, 10)
return repos, x.
Where("block_chain_status != ?", RepoBlockChainSuccess).
Find(&repos)
}

+ 26
- 1
models/user.go View File

@@ -6,6 +6,7 @@
package models

import (
"code.gitea.io/gitea/modules/blockchain"
"container/list"
"context"
"crypto/md5"
@@ -35,6 +36,7 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"

"github.com/go-resty/resty/v2"
"github.com/unknwon/com"
"golang.org/x/crypto/argon2"
"golang.org/x/crypto/bcrypt"
@@ -91,6 +93,8 @@ var (

// Characters prohibited in a user name (anything except A-Za-z0-9_.-)
alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)

restyClient *resty.Client
)

// User represents the object of individual and member of organization.
@@ -167,7 +171,11 @@ type User struct {
Theme string `xorm:"NOT NULL DEFAULT ''"`

//CloudBrain
Token string `xorm:"VARCHAR(1024)"`
Token string `xorm:"VARCHAR(1024)"`

//BlockChain
PublicKey string `xorm`
PrivateKey string `xorm`
}

// SearchOrganizationsOptions options to filter organizations
@@ -965,6 +973,15 @@ func CreateUser(u *User) (err error) {
return err
}

result, err := blockchain.CreateBlockchainAccount()
if err != nil {
log.Error("createBlockchainAccount failed:", err.Error())
return err
}

u.PublicKey = result.Payload["publickey"].(string)
u.PrivateKey = result.Payload["privatekey"].(string)

sess := x.NewSession()
defer sess.Close()
if err = sess.Begin(); err != nil {
@@ -2025,3 +2042,11 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
}
return nil
}

func GetBlockChainUnSuccessUsers() ([]*User, error) {
users := make([]*User, 0, 10)
err := x.Where("public_key is null").
Or("private_key is null").
Find(&users)
return users, err
}

+ 14
- 0
modules/blockchain/blockchain.go View File

@@ -0,0 +1,14 @@
package blockchain

const (
Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"`
CodeMountPath = "/code"
DataSetMountPath = "/dataset"
ModelMountPath = "/model"
BenchMarkMountPath = "/benchmark"
TaskInfoName = "/taskInfo"

SubTaskName = "task1"


)

+ 150
- 0
modules/blockchain/resty.go View File

@@ -0,0 +1,150 @@
package blockchain

import (
"fmt"

"code.gitea.io/gitea/modules/setting"
"github.com/go-resty/resty/v2"
)

var (
restyClient *resty.Client
)

const (
UrlCreateAccount = "createAccount"
UrlGetBalance = "getBalance"
UrlNewRepo = "newRepo"
UrlContribute = "contribute"

ActionCommit = "commit"

Success = 0
)

type CreateAccountResult struct {
Code int `json:"code"`
Msg string `json:"message"`
Payload map[string]interface{} `json:"data"`
}

type GetBalanceResult struct {
Code int `json:"code"`
Msg string `json:"message"`
Payload map[string]interface{} `json:"data"`
}

type NewRepoResult struct {
Code int `json:"code"`
Msg string `json:"message"`
//Data string `json:"data"`
}

type ContributeResult struct {
Code int `json:"code"`
Msg string `json:"message"`
Payload map[string]interface{} `json:"data"`
}

func getRestyClient() *resty.Client {
if restyClient == nil {
restyClient = resty.New()
}
return restyClient
}

func CreateBlockchainAccount() (*CreateAccountResult, error) {
client := getRestyClient()
var result CreateAccountResult

res, err := client.R().
SetHeader("Content-Type", "application/json").
SetResult(&result).
Get(setting.BlockChainHost + UrlCreateAccount)

if err != nil {
return nil, fmt.Errorf("resty create account: %s", err)
}

if result.Code != Success {
return &result, fmt.Errorf("CreateAccount err: %s", res.String())
}

return &result, nil
}

func NewRepo(repoID, publicKey, repoName string) (*NewRepoResult, error) {
client := getRestyClient()
var result NewRepoResult

res, err := client.R().
SetHeader("Accept", "application/json").
SetQueryParams(map[string]string{
"repoId" : repoID,
"creator" : publicKey,
"repoName" : repoName,
}).
SetResult(&result).
Get(setting.BlockChainHost + UrlNewRepo)

if err != nil {
return nil, fmt.Errorf("resty newRepo: %v", err)
}

if result.Code != Success {
return &result, fmt.Errorf("newRepo err: %s", res.String())
}

return &result, nil
}

func GetBalance(contractAddress, contributor string) (*GetBalanceResult, error) {
client := getRestyClient()
var result GetBalanceResult

res, err := client.R().
SetHeader("Accept", "application/json").
SetQueryParams(map[string]string{
"contractAddress" : contractAddress,
"contributor" : contributor,
}).
SetResult(&result).
Get(setting.BlockChainHost + UrlGetBalance)

if err != nil {
return nil, fmt.Errorf("resty getBalance: %v", err)
}

if result.Code != Success {
return &result, fmt.Errorf("getBalance err: %s", res.String())
}

return &result, nil
}

func Contribute(contractAddress, contributor, action, commitId string, codeLine int) (*ContributeResult, error) {
client := getRestyClient()
var result ContributeResult

res, err := client.R().
SetHeader("Accept", "application/json").
SetQueryParams(map[string]string{
"contractAddress" : contractAddress,
"contributor" : contributor,
"action" : action,
"commitId": commitId,
"amount": string(codeLine),
}).
SetResult(&result).
Get(setting.BlockChainHost + UrlContribute)

if err != nil {
return nil, fmt.Errorf("resty contribute: %v", err)
}

if result.Code != Success {
return &result, fmt.Errorf("contribute err: %s", res.String())
}

return &result, nil
}

+ 8
- 0
modules/setting/setting.go View File

@@ -443,6 +443,10 @@ var (
IsBenchmarkEnabled bool
BenchmarkCode string
BenchmarkServerHost string

//blockchain config
BlockChainHost string
CommitValidDate string
)

// DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1123,6 +1127,10 @@ func NewContext() {
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git")
BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/")

sec = Cfg.Section("blockchain")
BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/")
CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15")
}

func loadInternalToken(sec *ini.Section) string {


+ 13
- 0
modules/timer/timer.go View File

@@ -8,7 +8,20 @@ import (

func init() {
c := cron.New()

spec := "*/10 * * * *"
c.AddFunc(spec, repo.HandleUnDecompressAttachment)

specCheckBlockChainUserSuccess := "*/10 * * * *"
c.AddFunc(specCheckBlockChainUserSuccess, repo.HandleBlockChainUnSuccessUsers)

specCheckRepoBlockChainSuccess := "*/5 * * * *"
c.AddFunc(specCheckRepoBlockChainSuccess, repo.HandleBlockChainUnSuccessRepos)

specCheckUnTransformedActions := "*/1 * * * *"
c.AddFunc(specCheckUnTransformedActions, repo.HandleUnTransformedActions)

specCheckBlockChainCommitSuccess := "*/3 * * * *"
c.AddFunc(specCheckBlockChainCommitSuccess, repo.HandleBlockChainUnSuccessCommits)
c.Start()
}

+ 1
- 1
modules/worker/task.go View File

@@ -14,7 +14,7 @@ const (
)

func SendDecompressTask(ctx context.Context, uuid string) error {
args := []tasks.Arg{{Name: "uuid", Type: "string", Value: uuid}}
args := []tasks.Arg{{Name: "uuid", Type: "string", Value: uuid},{}}
task, err := tasks.NewSignature(DecompressTaskName, args)
if err != nil {
log.Error("NewSignature failed:", err.Error())


+ 250
- 0
routers/repo/blockchain.go View File

@@ -0,0 +1,250 @@
package repo

import (
"code.gitea.io/gitea/modules/repository"
"encoding/json"
"strconv"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/blockchain"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
)

type BlockChainInitNotify struct {
RepoId int64 `json:"repoId"`
ContractAddress string `json:"contractAddress"`
}

type BlockChainCommitNotify struct {
CommitID string `json:"commitId"`
TransactionHash string `json:"txHash"`
}

func HandleBlockChainInitNotify(ctx *context.Context) {
var req BlockChainInitNotify
data, _ := ctx.Req.Body().Bytes()
json.Unmarshal(data, &req)

repo, err := models.GetRepositoryByID(req.RepoId)
if err != nil {
log.Error("GetRepositoryByID failed:", err.Error())
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "internal error",
})
return
}

if repo.BlockChainStatus == models.RepoBlockChainSuccess && len(repo.ContractAddress) != 0 {
log.Error("the repo has been RepoBlockChainSuccess:", req.RepoId)
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "the repo has been RepoBlockChainSuccess",
})
return
}

repo.BlockChainStatus = models.RepoBlockChainSuccess
repo.ContractAddress = req.ContractAddress

if err = models.UpdateRepositoryCols(repo, "block_chain_status", "contract_address"); err != nil {
log.Error("UpdateRepositoryCols failed:", err.Error())
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "internal error",
})
return
}

ctx.JSON(200, map[string]string{
"code": "0",
"message": "",
})
}

func HandleBlockChainCommitNotify(ctx *context.Context) {
var req BlockChainCommitNotify
data, _ := ctx.Req.Body().Bytes()
if err := json.Unmarshal(data, &req); err != nil {
log.Error("json.Unmarshal failed:", err.Error())
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "response data error",
})
return
}

blockChain, err := models.GetBlockChainByCommitID(req.CommitID)
if err != nil {
log.Error("GetRepositoryByID failed:", err.Error())
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "internal error",
})
return
}

if blockChain.Status == models.BlockChainCommitSuccess {
log.Error("the commit has been BlockChainCommitReady:", blockChain.RepoID)
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "the commit has been BlockChainCommitReady",
})
return
}

blockChain.Status = models.BlockChainCommitSuccess
blockChain.TransactionHash = req.TransactionHash

if err = models.UpdateBlockChainCols(blockChain, "status", "transaction_hash"); err != nil {
log.Error("UpdateBlockChainCols failed:", err.Error())
ctx.JSON(200, map[string]string{
"code" : "-1",
"message" : "internal error",
})
return
}

ctx.JSON(200, map[string]string{
"code": "0",
"message": "",
})
}

func HandleBlockChainUnSuccessRepos() {
repos, err := models.GetBlockChainUnSuccessRepos()
if err != nil {
log.Error("GetBlockChainUnSuccessRepos failed:", err.Error())
return
}

for _, repo := range repos {
err = repo.GetOwner()
if err != nil {
log.Error("GetOwner(%s) failed:%v", repo.Name, err)
continue
}
if len(repo.Owner.PrivateKey) == 0 || len(repo.Owner.PublicKey) == 0 {
log.Error("the user has not been init in block_chain:", repo.Owner.Name)
continue
}
strRepoID := strconv.FormatInt(repo.ID, 10)
log.Info(strRepoID)
_, err = blockchain.NewRepo(strRepoID, repo.Owner.PublicKey, repo.Name)
if err != nil {
log.Error("blockchain.NewRepo(%s) failed:%v", strRepoID, err)
}
}

return
}

func HandleBlockChainUnSuccessCommits() {
blockChains, err := models.GetBlockChainUnSuccessCommits()
if err != nil {
log.Error("GetBlockChainUnSuccessCommits failed:", err.Error())
return
}

for _, block_chain := range blockChains {
_, err = blockchain.Contribute(block_chain.ContractAddress, block_chain.Contributor, blockchain.ActionCommit, block_chain.CommitID, int(block_chain.Amount))
if err != nil {
log.Error("blockchain.Contribute(%s) failed:%v", block_chain.CommitID, err)
}
}

return
}

func HandleBlockChainUnSuccessUsers() {
users, err := models.GetBlockChainUnSuccessUsers()
if err != nil {
log.Error("GetBlockChainUnSuccessUsers failed:", err.Error())
return
}

for _, user := range users {
result, err := blockchain.CreateBlockchainAccount()
if err != nil {
log.Error("blockchain.CreateBlockchainAccount(%s) failed:%v", user.Name, err)
continue
}

user.PublicKey = result.Payload["publickey"].(string)
user.PrivateKey = result.Payload["privatekey"].(string)

models.UpdateUser(user)
}

return
}

func HandleUnTransformedActions() {
actions, err := models.GetUnTransformedActions()
if err != nil {
log.Error("GetUnTransformedActions failed:", err.Error())
return
}

isTransformed := true

for _, action := range actions {
var content repository.PushCommits
err = json.Unmarshal([]byte(action.Content), &content)
if err != nil {
isTransformed = false
log.Error("json.Unmarshal action.Content(%s) failed:%v", action.Content, err)
break
}

repo, err := models.GetRepositoryByID(action.RepoID)
if err != nil {
isTransformed = false
log.Error("GetRepositoryByID(%d) failed:%v", action.RepoID, err)
break
}

if repo.ContractAddress == "" {
isTransformed = false
log.Error("the repo(%s) has not been initialized in block_chain", repo.Name)
break
}

for _, commit := range content.Commits {
_, err = models.GetBlockChainByCommitID(commit.Sha1)
if err == nil {
log.Info("the commit(%s) has been transformed", commit.Sha1)
continue
}

user, err := models.GetUserByName(commit.CommitterName)
if err != nil {
isTransformed = false
log.Error("GetUserByName(%s) failed:%v", commit.CommitterName, err)
break
}

blockChain := models.BlockChain{
CommitID : commit.Sha1,
Contributor : user.PublicKey,
ContractAddress : repo.ContractAddress,
Status : models.BlockChainCommitInit,
Amount : 1,
UserID : action.UserID,
RepoID : action.RepoID,
}
_, err = models.InsertBlockChain(&blockChain)
if err != nil {
isTransformed = false
log.Error("InsertBlockChain(%s) failed:%v", commit.Sha1, err)
break
}
}

}

log.Info("", isTransformed)

return
}

+ 5
- 0
routers/routes/routes.go View File

@@ -544,6 +544,11 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/action/:action", user.Action)
}, reqSignIn)

m.Group("/blockchain", func() {
m.Post("/init_notify", repo.HandleBlockChainInitNotify)
m.Post("/commit_notify", repo.HandleBlockChainCommitNotify)
})

if macaron.Env == macaron.DEV {
m.Get("/template/*", dev.TemplatePreview)
}


Loading…
Cancel
Save