From ada49fcd8db26fe2f493138d434c371ec93b47cd Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 2 Dec 2021 09:19:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=84=E7=90=86github=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/migrations/github.go | 62 ++++++++++++++++++++++++++++++++++++++----- modules/repository/repo.go | 24 ++++++++++++++--- modules/setting/git.go | 39 ++++++++++++++------------- modules/setting/migrations.go | 6 +++++ 4 files changed, 104 insertions(+), 27 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index b6bdab09f..bee21fb54 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -13,6 +13,9 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/setting" + "golang.org/x/net/proxy" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/structs" @@ -98,13 +101,41 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith ) client = oauth2.NewClient(downloader.ctx, ts) } else { - client = &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return nil, nil + if setting.Migrations.Proxy != "" { + contextDialer, err := getProxyDialContext() + if err != nil { + log.Warn("Failed to use proxy for Github.", err) + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + }, + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + DialContext: contextDialer.DialContext, + }, + } + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, }, - }, + } } } } @@ -112,6 +143,25 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +func getProxyDialContext() (proxy.ContextDialer, error) { + authInfo := &proxy.Auth{ + setting.Migrations.Username, + setting.Migrations.Password, + } + dialSocksProxy, err := proxy.SOCKS5("tcp", setting.Migrations.Proxy, authInfo, proxy.Direct) + + if err != nil { + return nil, err + } + if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { + return contextDialer, nil + + } else { + return nil, fmt.Errorf("It is not a valiad dialer.") + } + +} + // SetContext set context func (g *GithubDownloaderV3) SetContext(ctx context.Context) { g.ctx = ctx diff --git a/modules/repository/repo.go b/modules/repository/repo.go index d57b16c91..3268cce60 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -6,6 +6,7 @@ package repository import ( "fmt" + "net/url" "os" "path" "strings" @@ -54,29 +55,34 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo.NumWatches = 1 } - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + migrateTimeout := getMigrateTimeout(opts.CloneAddr) var err error if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - + log.Info("clone begin:" + opts.CloneAddr) if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, }); err != nil { + log.Warn("clone err") return repo, fmt.Errorf("Clone: %v", err) } + log.Info("clone end:" + opts.CloneAddr) + if opts.Wiki { + log.Info("test wiki path begin") wikiPath := models.WikiPath(u.Name, opts.RepoName) wikiRemotePath := wikiRemoteURL(opts.CloneAddr) + log.Info("test wiki path end") if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } - + log.Info("wiki clone begin") if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -88,6 +94,7 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } } + log.Info("wiki clone end") } } @@ -137,9 +144,20 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo, err = CleanUpMigrateInfo(repo) } + log.Info("clone all end:" + opts.CloneAddr) + return repo, err } +func getMigrateTimeout(urlClone string) time.Duration { + u, err := url.Parse(urlClone) + if err == nil && strings.EqualFold(u.Host, "github.com") { + return time.Duration(setting.Git.Timeout.GitHubMigrate) * time.Second + } + return time.Duration(setting.Git.Timeout.Migrate) * time.Second + +} + // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(configPath string) error { diff --git a/modules/setting/git.go b/modules/setting/git.go index f83758f38..306febc78 100644 --- a/modules/setting/git.go +++ b/modules/setting/git.go @@ -27,12 +27,13 @@ var ( EnableAutoGitWireProtocol bool PullRequestPushMessage bool Timeout struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` } `ini:"git.timeout"` }{ DisableDiffHighlight: false, @@ -45,19 +46,21 @@ var ( EnableAutoGitWireProtocol: true, PullRequestPushMessage: true, Timeout: struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` }{ - Default: int(git.DefaultCommandExecutionTimeout / time.Second), - Migrate: 600, - Mirror: 300, - Clone: 300, - Pull: 300, - GC: 60, + Default: int(git.DefaultCommandExecutionTimeout / time.Second), + Migrate: 900, + GitHubMigrate: 1800, + Mirror: 1200, + Clone: 300, + Pull: 300, + GC: 60, }, } ) diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 51d6bbcf1..0a948cd1f 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -9,6 +9,9 @@ var ( Migrations = struct { MaxAttempts int RetryBackoff int + Proxy string + Username string + Password string }{ MaxAttempts: 3, RetryBackoff: 3, @@ -19,4 +22,7 @@ func newMigrationsService() { sec := Cfg.Section("migrations") Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) + Migrations.Proxy = sec.Key("Proxy").MustString("") + Migrations.Username = sec.Key("Username").MustString("") + Migrations.Password = sec.Key("Password").MustString("") }