diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index d01866078..13a3d4aeb 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1065,3 +1065,7 @@ HOST = http://192.168.202.90:3366/ HOST = http://192.168.207.34:39987 USER = cW4cMtH24eoWPE7X PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC + +[blockchain] +HOST = http://192.168.207.84:3002/ +COMMIT_VALID_DATE = 2021-01-15 diff --git a/models/action.go b/models/action.go old mode 100644 new mode 100755 index fd49c6d4e..a29ed343d --- a/models/action.go +++ b/models/action.go @@ -67,6 +67,7 @@ type Action struct { IsDeleted bool `xorm:"INDEX NOT NULL DEFAULT false"` RefName string IsPrivate bool `xorm:"INDEX NOT NULL DEFAULT false"` + IsTransformed bool `xorm:"INDEX NOT NULL DEFAULT false"` Content string `xorm:"TEXT"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } @@ -344,3 +345,13 @@ func GetFeeds(opts GetFeedsOptions) ([]*Action, error) { return actions, nil } + +func GetUnTransformedActions() ([]*Action, error) { + actions := make([]*Action, 0, 10) + err := x.Where("op_type = ?", ActionCommitRepo). + And("content != ''"). + And("is_transformed = ?", false). + And("to_timestamp(created_unix) >= ?", setting.CommitValidDate). + Find(&actions) + return actions, err +} diff --git a/models/blockchain.go b/models/blockchain.go new file mode 100755 index 000000000..a84636a5b --- /dev/null +++ b/models/blockchain.go @@ -0,0 +1,86 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "fmt" + "time" +) + +type BlockChainCommitStatus int +const ( + BlockChainCommitInit BlockChainCommitStatus = iota + BlockChainCommitSuccess + BlockChainCommitFailed +) +type BlockChain struct { + ID int64 `xorm:"pk autoincr"` + CommitID string `xorm:"INDEX NOT NULL"` + Contributor string `xorm:"INDEX NOT NULL"` + ContractAddress string `xorm:"INDEX NOT NULL"` + Status BlockChainCommitStatus `xorm:"INDEX NOT NULL DEFAULT 0"` + Amount int64 `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + TransactionHash string `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + DeletedAt time.Time `xorm:"deleted"` + + User *User `xorm:"-"` + Repo *Repository `xorm:"-"` +} + +func getBlockChainByID(e Engine, id int64) (*BlockChain, error) { + blockChain := new(BlockChain) + has, err := e.ID(id).Get(blockChain) + if err != nil { + return nil, err + } else if !has { + return nil, fmt.Errorf("get block_chain by id failed(%d)", id) + } + return blockChain, nil +} + +func GetBlockChainByID(id int64) (*BlockChain, error) { + return getBlockChainByID(x, id) +} + +func getBlockChainByCommitID(e Engine, commitID string) (*BlockChain, error) { + blockChain := new(BlockChain) + has, err := e.Where("commit_id = ?", commitID).Get(blockChain) + if err != nil { + return nil, err + } else if !has { + return nil, fmt.Errorf("get block_chain by commitID failed(%s)", commitID) + } + return blockChain, nil +} + +func GetBlockChainByCommitID(commitID string) (*BlockChain, error) { + return getBlockChainByCommitID(x, commitID) +} + +func updateBlockChainCols(e Engine, blockChain *BlockChain, cols ...string) error { + _, err := e.ID(blockChain.ID).Cols(cols...).Update(blockChain) + return err +} + +func UpdateBlockChainCols(blockChain *BlockChain, cols ...string) error { + return updateBlockChainCols(x, blockChain, cols...) +} + +func GetBlockChainUnSuccessCommits() ([]*BlockChain, error) { + blockChains := make([]*BlockChain, 0, 10) + return blockChains, x. + Where("status != ?", BlockChainCommitSuccess). + Find(&blockChains) +} + +func InsertBlockChain(blockChain *BlockChain) (_ *BlockChain, err error) { + + if _, err := x.Insert(blockChain); err != nil { + return nil, err + } + + return blockChain, nil +} diff --git a/models/models.go b/models/models.go index b709a2d28..4f7683774 100755 --- a/models/models.go +++ b/models/models.go @@ -128,6 +128,7 @@ func init() { new(Dataset), new(Cloudbrain), new(FileChunk), + new(BlockChain), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 52a194ab0..7cda4ce8a 100755 --- a/models/repo.go +++ b/models/repo.go @@ -6,6 +6,7 @@ package models import ( + "code.gitea.io/gitea/modules/blockchain" "context" "crypto/md5" "errors" @@ -134,6 +135,7 @@ func NewRepoContext() { // RepositoryStatus defines the status of repository type RepositoryStatus int +type RepoBlockChainStatus int // all kinds of RepositoryStatus const ( @@ -141,6 +143,12 @@ const ( RepositoryBeingMigrated // repository is migrating ) +const ( + RepoBlockChainInit RepoBlockChainStatus = iota + RepoBlockChainSuccess + RepoBlockChainFailed +) + // Repository represents a git repository. type Repository struct { ID int64 `xorm:"pk autoincr"` @@ -195,6 +203,11 @@ type Repository struct { // Avatar: ID(10-20)-md5(32) - must fit into 64 symbols Avatar string `xorm:"VARCHAR(64)"` + //blockchain + ContractAddress string `xorm:"INDEX"` + Balance int64 `xorm:"NOT NULL DEFAULT 0"` + BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } @@ -1051,6 +1064,11 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository) (err error return err } + _, err = blockchain.NewRepo(string(repo.ID), u.PublicKey, repo.Name) + if err != nil { + log.Error("newRepo failed:", err.Error()) + } + // insert units for repo var units = make([]RepoUnit, 0, len(DefaultRepoUnits)) for _, tp := range DefaultRepoUnits { @@ -2388,3 +2406,10 @@ func updateRepositoryCols(e Engine, repo *Repository, cols ...string) error { func UpdateRepositoryCols(repo *Repository, cols ...string) error { return updateRepositoryCols(x, repo, cols...) } + +func GetBlockChainUnSuccessRepos() ([]*Repository, error) { + repos := make([]*Repository, 0, 10) + return repos, x. + Where("block_chain_status != ?", RepoBlockChainSuccess). + Find(&repos) +} diff --git a/models/user.go b/models/user.go index 29e1a5d9f..2b1195179 100755 --- a/models/user.go +++ b/models/user.go @@ -6,6 +6,7 @@ package models import ( + "code.gitea.io/gitea/modules/blockchain" "container/list" "context" "crypto/md5" @@ -35,6 +36,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "github.com/go-resty/resty/v2" "github.com/unknwon/com" "golang.org/x/crypto/argon2" "golang.org/x/crypto/bcrypt" @@ -91,6 +93,8 @@ var ( // Characters prohibited in a user name (anything except A-Za-z0-9_.-) alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) + + restyClient *resty.Client ) // User represents the object of individual and member of organization. @@ -167,7 +171,11 @@ type User struct { Theme string `xorm:"NOT NULL DEFAULT ''"` //CloudBrain - Token string `xorm:"VARCHAR(1024)"` + Token string `xorm:"VARCHAR(1024)"` + + //BlockChain + PublicKey string `xorm` + PrivateKey string `xorm` } // SearchOrganizationsOptions options to filter organizations @@ -965,6 +973,15 @@ func CreateUser(u *User) (err error) { return err } + result, err := blockchain.CreateBlockchainAccount() + if err != nil { + log.Error("createBlockchainAccount failed:", err.Error()) + return err + } + + u.PublicKey = result.Payload["publickey"].(string) + u.PrivateKey = result.Payload["privatekey"].(string) + sess := x.NewSession() defer sess.Close() if err = sess.Begin(); err != nil { @@ -2025,3 +2042,11 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { } return nil } + +func GetBlockChainUnSuccessUsers() ([]*User, error) { + users := make([]*User, 0, 10) + err := x.Where("public_key is null"). + Or("private_key is null"). + Find(&users) + return users, err +} diff --git a/modules/blockchain/blockchain.go b/modules/blockchain/blockchain.go new file mode 100755 index 000000000..6ebf701da --- /dev/null +++ b/modules/blockchain/blockchain.go @@ -0,0 +1,14 @@ +package blockchain + +const ( + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + CodeMountPath = "/code" + DataSetMountPath = "/dataset" + ModelMountPath = "/model" + BenchMarkMountPath = "/benchmark" + TaskInfoName = "/taskInfo" + + SubTaskName = "task1" + + +) diff --git a/modules/blockchain/resty.go b/modules/blockchain/resty.go new file mode 100755 index 000000000..060d63144 --- /dev/null +++ b/modules/blockchain/resty.go @@ -0,0 +1,150 @@ +package blockchain + +import ( + "fmt" + + "code.gitea.io/gitea/modules/setting" + "github.com/go-resty/resty/v2" +) + +var ( + restyClient *resty.Client +) + +const ( + UrlCreateAccount = "createAccount" + UrlGetBalance = "getBalance" + UrlNewRepo = "newRepo" + UrlContribute = "contribute" + + ActionCommit = "commit" + + Success = 0 +) + +type CreateAccountResult struct { + Code int `json:"code"` + Msg string `json:"message"` + Payload map[string]interface{} `json:"data"` +} + +type GetBalanceResult struct { + Code int `json:"code"` + Msg string `json:"message"` + Payload map[string]interface{} `json:"data"` +} + +type NewRepoResult struct { + Code int `json:"code"` + Msg string `json:"message"` + //Data string `json:"data"` +} + +type ContributeResult struct { + Code int `json:"code"` + Msg string `json:"message"` + Payload map[string]interface{} `json:"data"` +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + } + return restyClient +} + +func CreateBlockchainAccount() (*CreateAccountResult, error) { + client := getRestyClient() + var result CreateAccountResult + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetResult(&result). + Get(setting.BlockChainHost + UrlCreateAccount) + + if err != nil { + return nil, fmt.Errorf("resty create account: %s", err) + } + + if result.Code != Success { + return &result, fmt.Errorf("CreateAccount err: %s", res.String()) + } + + return &result, nil +} + +func NewRepo(repoID, publicKey, repoName string) (*NewRepoResult, error) { + client := getRestyClient() + var result NewRepoResult + + res, err := client.R(). + SetHeader("Accept", "application/json"). + SetQueryParams(map[string]string{ + "repoId" : repoID, + "creator" : publicKey, + "repoName" : repoName, + }). + SetResult(&result). + Get(setting.BlockChainHost + UrlNewRepo) + + if err != nil { + return nil, fmt.Errorf("resty newRepo: %v", err) + } + + if result.Code != Success { + return &result, fmt.Errorf("newRepo err: %s", res.String()) + } + + return &result, nil +} + +func GetBalance(contractAddress, contributor string) (*GetBalanceResult, error) { + client := getRestyClient() + var result GetBalanceResult + + res, err := client.R(). + SetHeader("Accept", "application/json"). + SetQueryParams(map[string]string{ + "contractAddress" : contractAddress, + "contributor" : contributor, + }). + SetResult(&result). + Get(setting.BlockChainHost + UrlGetBalance) + + if err != nil { + return nil, fmt.Errorf("resty getBalance: %v", err) + } + + if result.Code != Success { + return &result, fmt.Errorf("getBalance err: %s", res.String()) + } + + return &result, nil +} + +func Contribute(contractAddress, contributor, action, commitId string, codeLine int) (*ContributeResult, error) { + client := getRestyClient() + var result ContributeResult + + res, err := client.R(). + SetHeader("Accept", "application/json"). + SetQueryParams(map[string]string{ + "contractAddress" : contractAddress, + "contributor" : contributor, + "action" : action, + "commitId": commitId, + "amount": string(codeLine), + }). + SetResult(&result). + Get(setting.BlockChainHost + UrlContribute) + + if err != nil { + return nil, fmt.Errorf("resty contribute: %v", err) + } + + if result.Code != Success { + return &result, fmt.Errorf("contribute err: %s", res.String()) + } + + return &result, nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a71617e8d..f3888b9e7 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -443,6 +443,10 @@ var ( IsBenchmarkEnabled bool BenchmarkCode string BenchmarkServerHost string + + //blockchain config + BlockChainHost string + CommitValidDate string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1123,6 +1127,10 @@ func NewContext() { IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + + sec = Cfg.Section("blockchain") + BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") + CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") } func loadInternalToken(sec *ini.Section) string { diff --git a/modules/timer/timer.go b/modules/timer/timer.go index d0a850bca..60aaa7498 100755 --- a/modules/timer/timer.go +++ b/modules/timer/timer.go @@ -8,7 +8,20 @@ import ( func init() { c := cron.New() + spec := "*/10 * * * *" c.AddFunc(spec, repo.HandleUnDecompressAttachment) + + specCheckBlockChainUserSuccess := "*/10 * * * *" + c.AddFunc(specCheckBlockChainUserSuccess, repo.HandleBlockChainUnSuccessUsers) + + specCheckRepoBlockChainSuccess := "*/5 * * * *" + c.AddFunc(specCheckRepoBlockChainSuccess, repo.HandleBlockChainUnSuccessRepos) + + specCheckUnTransformedActions := "*/1 * * * *" + c.AddFunc(specCheckUnTransformedActions, repo.HandleUnTransformedActions) + + specCheckBlockChainCommitSuccess := "*/3 * * * *" + c.AddFunc(specCheckBlockChainCommitSuccess, repo.HandleBlockChainUnSuccessCommits) c.Start() } diff --git a/modules/worker/task.go b/modules/worker/task.go index 073b16b92..48ba32c3e 100755 --- a/modules/worker/task.go +++ b/modules/worker/task.go @@ -14,7 +14,7 @@ const ( ) func SendDecompressTask(ctx context.Context, uuid string) error { - args := []tasks.Arg{{Name: "uuid", Type: "string", Value: uuid}} + args := []tasks.Arg{{Name: "uuid", Type: "string", Value: uuid},{}} task, err := tasks.NewSignature(DecompressTaskName, args) if err != nil { log.Error("NewSignature failed:", err.Error()) diff --git a/routers/repo/blockchain.go b/routers/repo/blockchain.go new file mode 100755 index 000000000..9aa705574 --- /dev/null +++ b/routers/repo/blockchain.go @@ -0,0 +1,250 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/repository" + "encoding/json" + "strconv" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/blockchain" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +type BlockChainInitNotify struct { + RepoId int64 `json:"repoId"` + ContractAddress string `json:"contractAddress"` +} + +type BlockChainCommitNotify struct { + CommitID string `json:"commitId"` + TransactionHash string `json:"txHash"` +} + +func HandleBlockChainInitNotify(ctx *context.Context) { + var req BlockChainInitNotify + data, _ := ctx.Req.Body().Bytes() + json.Unmarshal(data, &req) + + repo, err := models.GetRepositoryByID(req.RepoId) + if err != nil { + log.Error("GetRepositoryByID failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "internal error", + }) + return + } + + if repo.BlockChainStatus == models.RepoBlockChainSuccess && len(repo.ContractAddress) != 0 { + log.Error("the repo has been RepoBlockChainSuccess:", req.RepoId) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "the repo has been RepoBlockChainSuccess", + }) + return + } + + repo.BlockChainStatus = models.RepoBlockChainSuccess + repo.ContractAddress = req.ContractAddress + + if err = models.UpdateRepositoryCols(repo, "block_chain_status", "contract_address"); err != nil { + log.Error("UpdateRepositoryCols failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "internal error", + }) + return + } + + ctx.JSON(200, map[string]string{ + "code": "0", + "message": "", + }) +} + +func HandleBlockChainCommitNotify(ctx *context.Context) { + var req BlockChainCommitNotify + data, _ := ctx.Req.Body().Bytes() + if err := json.Unmarshal(data, &req); err != nil { + log.Error("json.Unmarshal failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "response data error", + }) + return + } + + blockChain, err := models.GetBlockChainByCommitID(req.CommitID) + if err != nil { + log.Error("GetRepositoryByID failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "internal error", + }) + return + } + + if blockChain.Status == models.BlockChainCommitSuccess { + log.Error("the commit has been BlockChainCommitReady:", blockChain.RepoID) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "the commit has been BlockChainCommitReady", + }) + return + } + + blockChain.Status = models.BlockChainCommitSuccess + blockChain.TransactionHash = req.TransactionHash + + if err = models.UpdateBlockChainCols(blockChain, "status", "transaction_hash"); err != nil { + log.Error("UpdateBlockChainCols failed:", err.Error()) + ctx.JSON(200, map[string]string{ + "code" : "-1", + "message" : "internal error", + }) + return + } + + ctx.JSON(200, map[string]string{ + "code": "0", + "message": "", + }) +} + +func HandleBlockChainUnSuccessRepos() { + repos, err := models.GetBlockChainUnSuccessRepos() + if err != nil { + log.Error("GetBlockChainUnSuccessRepos failed:", err.Error()) + return + } + + for _, repo := range repos { + err = repo.GetOwner() + if err != nil { + log.Error("GetOwner(%s) failed:%v", repo.Name, err) + continue + } + if len(repo.Owner.PrivateKey) == 0 || len(repo.Owner.PublicKey) == 0 { + log.Error("the user has not been init in block_chain:", repo.Owner.Name) + continue + } + strRepoID := strconv.FormatInt(repo.ID, 10) + log.Info(strRepoID) + _, err = blockchain.NewRepo(strRepoID, repo.Owner.PublicKey, repo.Name) + if err != nil { + log.Error("blockchain.NewRepo(%s) failed:%v", strRepoID, err) + } + } + + return +} + +func HandleBlockChainUnSuccessCommits() { + blockChains, err := models.GetBlockChainUnSuccessCommits() + if err != nil { + log.Error("GetBlockChainUnSuccessCommits failed:", err.Error()) + return + } + + for _, block_chain := range blockChains { + _, err = blockchain.Contribute(block_chain.ContractAddress, block_chain.Contributor, blockchain.ActionCommit, block_chain.CommitID, int(block_chain.Amount)) + if err != nil { + log.Error("blockchain.Contribute(%s) failed:%v", block_chain.CommitID, err) + } + } + + return +} + +func HandleBlockChainUnSuccessUsers() { + users, err := models.GetBlockChainUnSuccessUsers() + if err != nil { + log.Error("GetBlockChainUnSuccessUsers failed:", err.Error()) + return + } + + for _, user := range users { + result, err := blockchain.CreateBlockchainAccount() + if err != nil { + log.Error("blockchain.CreateBlockchainAccount(%s) failed:%v", user.Name, err) + continue + } + + user.PublicKey = result.Payload["publickey"].(string) + user.PrivateKey = result.Payload["privatekey"].(string) + + models.UpdateUser(user) + } + + return +} + +func HandleUnTransformedActions() { + actions, err := models.GetUnTransformedActions() + if err != nil { + log.Error("GetUnTransformedActions failed:", err.Error()) + return + } + + isTransformed := true + + for _, action := range actions { + var content repository.PushCommits + err = json.Unmarshal([]byte(action.Content), &content) + if err != nil { + isTransformed = false + log.Error("json.Unmarshal action.Content(%s) failed:%v", action.Content, err) + break + } + + repo, err := models.GetRepositoryByID(action.RepoID) + if err != nil { + isTransformed = false + log.Error("GetRepositoryByID(%d) failed:%v", action.RepoID, err) + break + } + + if repo.ContractAddress == "" { + isTransformed = false + log.Error("the repo(%s) has not been initialized in block_chain", repo.Name) + break + } + + for _, commit := range content.Commits { + _, err = models.GetBlockChainByCommitID(commit.Sha1) + if err == nil { + log.Info("the commit(%s) has been transformed", commit.Sha1) + continue + } + + user, err := models.GetUserByName(commit.CommitterName) + if err != nil { + isTransformed = false + log.Error("GetUserByName(%s) failed:%v", commit.CommitterName, err) + break + } + + blockChain := models.BlockChain{ + CommitID : commit.Sha1, + Contributor : user.PublicKey, + ContractAddress : repo.ContractAddress, + Status : models.BlockChainCommitInit, + Amount : 1, + UserID : action.UserID, + RepoID : action.RepoID, + } + _, err = models.InsertBlockChain(&blockChain) + if err != nil { + isTransformed = false + log.Error("InsertBlockChain(%s) failed:%v", commit.Sha1, err) + break + } + } + + } + + log.Info("", isTransformed) + + return +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index f40b55e2b..3d797a1e7 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -544,6 +544,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/action/:action", user.Action) }, reqSignIn) + m.Group("/blockchain", func() { + m.Post("/init_notify", repo.HandleBlockChainInitNotify) + m.Post("/commit_notify", repo.HandleBlockChainCommitNotify) + }) + if macaron.Env == macaron.DEV { m.Get("/template/*", dev.TemplatePreview) }