From 56d040b8acfa8f6eabcb97bf719ba27921d5fc34 Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 12:11:38 +0100 Subject: [PATCH 1/4] Adding more error handling in dump cmd The dump cmd did not check the return value of the z.AddFile or z.AddDir when building the final archive. It caused the dump command to succeed even if an error occurred. The resulted dump archive could be corrupted/empty. (errors could be various: removal by a concurrent process, disk full, bugs in the dump cmd itself) --- cmd/dump.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 0bf385d06..279099dac 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -60,10 +60,18 @@ func runDump(ctx *cli.Context) { } workDir, _ := setting.WorkDir() - z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")) - z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")) - z.AddDir("custom", path.Join(workDir, "custom")) - z.AddDir("log", path.Join(workDir, "log")) + if err := z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")); err !=nil { + log.Fatalf("Fail to include gogs-repo.zip: %v", err) + } + if err := z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")); err !=nil { + log.Fatalf("Fail to include gogs-db.sql: %v", err) + } + if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { + log.Fatalf("Fail to include custom: %v", err) + } + if err := z.AddDir("log", path.Join(workDir, "log")); err !=nil { + log.Fatalf("Fail to include log: %v", err) + } // FIXME: SSH key file. if err = z.Close(); err != nil { os.Remove(fileName) From 37aad3389def5e89282f09c5ceea1cb71eccc972 Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 14:07:51 +0100 Subject: [PATCH 2/4] Using a tmp dir to generate db and repo dumps Using a tmp dir makes gogs dump more robust to concurrent runs. It also permits an easier cleaning of the tmp files (gogs-db.sql and gog-repo.zip) by just removing the tmp dir. As a side effect, it partially fix bugs on workdir. Previously, 'gogs dump' created the archives in the current directory, and tried to include these archives from the directory where the gogs binary lies. ex: if gogs binary is in /usr/bin/gogs, and gogs dump is run from /tmp/, /tmp/gog-repo.zip is created, but gogs dump tried to include /usr/bin/gogs-repo.zip. --- cmd/dump.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 279099dac..0c07f36cf 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -11,6 +11,8 @@ import ( "path" "time" + "io/ioutil" + "github.com/Unknwon/cae/zip" "github.com/codegangsta/cli" @@ -38,16 +40,23 @@ func runDump(ctx *cli.Context) { models.LoadConfigs() models.SetEngine() + TmpWorkDir, err := ioutil.TempDir(os.TempDir(), "gogs-dump-") + if err != nil { + log.Fatalf("Fail to create tmp work directory: %v", err) + } + log.Printf("Creating tmp work dir: %s", TmpWorkDir) + + reposDump := path.Join(TmpWorkDir, "gogs-repo.zip") + dbDump := path.Join(TmpWorkDir, "gogs-db.sql") + log.Printf("Dumping local repositories...%s", setting.RepoRootPath) zip.Verbose = ctx.Bool("verbose") - defer os.Remove("gogs-repo.zip") - if err := zip.PackTo(setting.RepoRootPath, "gogs-repo.zip", true); err != nil { + if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { log.Fatalf("Fail to dump local repositories: %v", err) } log.Printf("Dumping database...") - defer os.Remove("gogs-db.sql") - if err := models.DumpDatabase("gogs-db.sql"); err != nil { + if err := models.DumpDatabase(dbDump); err != nil { log.Fatalf("Fail to dump database: %v", err) } @@ -60,10 +69,10 @@ func runDump(ctx *cli.Context) { } workDir, _ := setting.WorkDir() - if err := z.AddFile("gogs-repo.zip", path.Join(workDir, "gogs-repo.zip")); err !=nil { + if err := z.AddFile("gogs-repo.zip", reposDump); err !=nil { log.Fatalf("Fail to include gogs-repo.zip: %v", err) } - if err := z.AddFile("gogs-db.sql", path.Join(workDir, "gogs-db.sql")); err !=nil { + if err := z.AddFile("gogs-db.sql", dbDump); err !=nil { log.Fatalf("Fail to include gogs-db.sql: %v", err) } if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { @@ -78,5 +87,7 @@ func runDump(ctx *cli.Context) { log.Fatalf("Fail to save %s: %v", fileName, err) } + log.Printf("Removing tmp work dir: %s", TmpWorkDir) + os.RemoveAll(TmpWorkDir) log.Println("Finish dumping!") } From 4138113c966bbfb37a4b14f59f42c6d73eaa3420 Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 15:08:50 +0100 Subject: [PATCH 3/4] Fix dump of log and custom directory in dump cmd Now, the dump cmd uses setting.CustomPath and setting.LogRootPath instead of setting.WorkDir which was kind of broken if the gogs binary was in a different directory than gogs data. Additionally, the backup of setting.CustomPath directory is only done if it exists. --- cmd/dump.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 0c07f36cf..e8354784c 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -68,17 +68,21 @@ func runDump(ctx *cli.Context) { log.Fatalf("Fail to create %s: %v", fileName, err) } - workDir, _ := setting.WorkDir() if err := z.AddFile("gogs-repo.zip", reposDump); err !=nil { log.Fatalf("Fail to include gogs-repo.zip: %v", err) } if err := z.AddFile("gogs-db.sql", dbDump); err !=nil { log.Fatalf("Fail to include gogs-db.sql: %v", err) } - if err := z.AddDir("custom", path.Join(workDir, "custom")); err !=nil { - log.Fatalf("Fail to include custom: %v", err) + customDir, err := os.Stat(setting.CustomPath) + if err == nil && customDir.IsDir() { + if err := z.AddDir("custom", setting.CustomPath); err !=nil { + log.Fatalf("Fail to include custom: %v", err) + } + } else { + log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath) } - if err := z.AddDir("log", path.Join(workDir, "log")); err !=nil { + if err := z.AddDir("log", setting.LogRootPath); err !=nil { log.Fatalf("Fail to include log: %v", err) } // FIXME: SSH key file. From 7e3fa5f1acef5e432cd68bde0a6aa5b9ae125976 Mon Sep 17 00:00:00 2001 From: kakwa Date: Sat, 28 Nov 2015 15:22:10 +0100 Subject: [PATCH 4/4] add name of the dump file in last log message --- cmd/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dump.go b/cmd/dump.go index e8354784c..90082667b 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -93,5 +93,5 @@ func runDump(ctx *cli.Context) { log.Printf("Removing tmp work dir: %s", TmpWorkDir) os.RemoveAll(TmpWorkDir) - log.Println("Finish dumping!") + log.Printf("Finish dumping in file %s", fileName) }