From 738bfbdb69eec25a4b66b73eb8992faf28faffc6 Mon Sep 17 00:00:00 2001 From: Itxaka Date: Tue, 20 Jun 2023 12:18:31 +0200 Subject: [PATCH] Simplify rsync implementation (#61) --- go.mod | 1 - pkg/constants/constants.go | 2 + pkg/elemental/elemental.go | 2 +- pkg/elemental/elemental_test.go | 35 ++++++++--------- pkg/utils/common.go | 68 ++++++++++++++++----------------- pkg/utils/utils_test.go | 14 ++++--- 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index 7d2a75d..93d522b 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/spf13/viper v1.8.1 github.com/twpayne/go-vfs v1.7.2 github.com/urfave/cli/v2 v2.25.1 - github.com/zloylos/grsync v1.7.0 golang.org/x/net v0.10.0 golang.org/x/oauth2 v0.7.0 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 6ec73d7..c130d1b 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -115,6 +115,8 @@ const ( Archx86 = "x86_64" ArchArm64 = "arm64" SignedShim = "shim.efi" + + Rsync = "rsync" ) func GetCloudInitPaths() []string { diff --git a/pkg/elemental/elemental.go b/pkg/elemental/elemental.go index 3b40f52..f60ef7c 100644 --- a/pkg/elemental/elemental.go +++ b/pkg/elemental/elemental.go @@ -369,7 +369,7 @@ func (e *Elemental) DumpSource(target string, imgSrc *v1.ImageSource) (info inte } } else if imgSrc.IsDir() { excludes := []string{"/mnt", "/proc", "/sys", "/dev", "/tmp", "/host", "/run"} - err = utils.SyncData(e.config.Logger, e.config.Fs, imgSrc.Value(), target, excludes...) + err = utils.SyncData(e.config.Logger, e.config.Runner, e.config.Fs, imgSrc.Value(), target, excludes...) if err != nil { return nil, err } diff --git a/pkg/elemental/elemental_test.go b/pkg/elemental/elemental_test.go index a76a21d..88e2d88 100644 --- a/pkg/elemental/elemental_test.go +++ b/pkg/elemental/elemental_test.go @@ -584,11 +584,6 @@ var _ = Describe("Elemental", Label("elemental"), func() { _, err := el.DeployImage(img, true) Expect(err).NotTo(BeNil()) }) - It("Fails copying the image if source does not exist", func() { - img.Source = v1.NewDirSrc("/welp") - _, err := el.DeployImage(img, true) - Expect(err).NotTo(BeNil()) - }) It("Fails unmounting the image after copying", func() { mounter.ErrorOnUnmount = true _, err := el.DeployImage(img, false) @@ -605,14 +600,23 @@ var _ = Describe("Elemental", Label("elemental"), func() { Expect(err).ShouldNot(HaveOccurred()) }) It("Copies files from a directory source", func() { - sourceDir, err := utils.TempDir(fs, "", "elemental") + rsyncCount := 0 + src := "" + dest := "" + runner.SideEffect = func(cmd string, args ...string) ([]byte, error) { + if cmd == constants.Rsync { + rsyncCount += 1 + src = args[len(args)-2] + dest = args[len(args)-1] + + } + return []byte{}, nil + } + _, err := e.DumpSource("/dest", v1.NewDirSrc("/source")) Expect(err).ShouldNot(HaveOccurred()) - _, err = e.DumpSource(destDir, v1.NewDirSrc(sourceDir)) - Expect(err).To(BeNil()) - }) - It("Fails if source directory does not exist", func() { - _, err := e.DumpSource(destDir, v1.NewDirSrc("/welp")) - Expect(err).ToNot(BeNil()) + Expect(rsyncCount).To(Equal(1)) + Expect(src).To(HaveSuffix("/source/")) + Expect(dest).To(HaveSuffix("/dest/")) }) It("Unpacks a docker image to target", Label("docker"), func() { _, err := e.DumpSource(destDir, v1.NewDockerSrc("docker/image:latest")) @@ -633,15 +637,12 @@ var _ = Describe("Elemental", Label("elemental"), func() { }) It("Copies image file to target", func() { sourceImg := "/source.img" + destFile := filepath.Join(destDir, "active.img") _, err := fs.Create(sourceImg) Expect(err).To(BeNil()) - destFile := filepath.Join(destDir, "active.img") - _, err = fs.Stat(destFile) - Expect(err).NotTo(BeNil()) _, err = e.DumpSource(destFile, v1.NewFileSrc(sourceImg)) Expect(err).To(BeNil()) - _, err = fs.Stat(destFile) - Expect(err).To(BeNil()) + Expect(runner.IncludesCmds([][]string{{constants.Rsync}})) }) It("Fails to copy, source file is not present", func() { _, err := e.DumpSource("whatever", v1.NewFileSrc("/source.img")) diff --git a/pkg/utils/common.go b/pkg/utils/common.go index e999849..bbab461 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -32,11 +32,9 @@ import ( "github.com/distribution/distribution/reference" "github.com/joho/godotenv" - "github.com/twpayne/go-vfs" - "github.com/zloylos/grsync" - cnst "github.com/kairos-io/kairos/v2/pkg/constants" v1 "github.com/kairos-io/kairos/v2/pkg/types/v1" + "github.com/twpayne/go-vfs" ) func CommandExists(command string) bool { @@ -156,7 +154,7 @@ func CreateDirStructure(fs v1.FS, target string) error { // SyncData rsync's source folder contents to a target folder content, // both are expected to exist beforehand. -func SyncData(log v1.Logger, fs v1.FS, source string, target string, excludes ...string) error { +func SyncData(log v1.Logger, runner v1.Runner, fs v1.FS, source string, target string, excludes ...string) error { if fs != nil { if s, err := fs.RawPath(source); err == nil { source = s @@ -174,46 +172,46 @@ func SyncData(log v1.Logger, fs v1.FS, source string, target string, excludes .. target = fmt.Sprintf("%s/", target) } - task := grsync.NewTask( - source, - target, - grsync.RsyncOptions{ - Quiet: false, - Archive: true, - XAttrs: true, - ACLs: true, - Exclude: excludes, - }, - ) + log.Infof("Starting rsync...") + args := []string{"--progress", "--partial", "--human-readable", "--archive", "--xattrs", "--acls"} + + for _, e := range excludes { + args = append(args, fmt.Sprintf("--exclude=%s", e)) + } + + args = append(args, source, target) + + done := displayProgress(log, 5*time.Second, "Syncing data...") + + _, err := runner.Run(cnst.Rsync, args...) + + close(done) + if err != nil { + log.Errorf("rsync finished with errors: %s", err.Error()) + return err + } + log.Info("Finished syncing") + + return nil +} + +func displayProgress(log v1.Logger, tick time.Duration, message string) chan bool { + ticker := time.NewTicker(tick) + done := make(chan bool) - quit := make(chan bool) go func() { for { select { - case <-quit: + case <-done: + ticker.Stop() return - case <-time.After(5 * time.Second): - state := task.State() - log.Debugf( - "progress rsync %s to %s: %.2f / rem. %d / tot. %d / sp. %s", - source, - target, - state.Progress, - state.Remain, - state.Total, - state.Speed, - ) + case <-ticker.C: + log.Debug(message) } } }() - err := task.Run() - quit <- true - if err != nil { - return fmt.Errorf("%w: %s", err, strings.Join([]string{task.Log().Stderr, task.Log().Stdout}, "\n")) - } - - return nil + return done } // Reboot reboots the system afater the given delay (in seconds) time passed. diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index d19df2d..0473874 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -53,6 +53,7 @@ var _ = Describe("Utils", Label("utils"), func() { var syscall *v1mock.FakeSyscall var client *v1mock.FakeHTTPClient var mounter *v1mock.ErrorMounter + var realRunner *v1.RealRunner var fs vfs.FS var cleanup func() @@ -62,6 +63,7 @@ var _ = Describe("Utils", Label("utils"), func() { mounter = v1mock.NewErrorMounter() client = &v1mock.FakeHTTPClient{} logger = v1.NewNullLogger() + realRunner = &v1.RealRunner{Logger: logger} // Ensure /tmp exists in the VFS fs, cleanup, _ = vfst.NewTestFS(nil) fs.Mkdir("/tmp", constants.DirPerm) @@ -455,7 +457,7 @@ var _ = Describe("Utils", Label("utils"), func() { _, _ = utils.TempFile(fs, sourceDir, "file*") } - Expect(utils.SyncData(logger, fs, sourceDir, destDir)).To(BeNil()) + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) filesDest, err := fs.ReadDir(destDir) Expect(err).To(BeNil()) @@ -486,7 +488,7 @@ var _ = Describe("Utils", Label("utils"), func() { _, _ = utils.TempFile(fs, sourceDir, "file*") } - Expect(utils.SyncData(logger, fs, sourceDir, destDir, "host", "run")).To(BeNil()) + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "host", "run")).To(BeNil()) filesDest, err := fs.ReadDir(destDir) Expect(err).To(BeNil()) @@ -524,7 +526,7 @@ var _ = Describe("Utils", Label("utils"), func() { utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "tmp", "host"), constants.DirPerm) - Expect(utils.SyncData(logger, fs, sourceDir, destDir, "/host", "/run")).To(BeNil()) + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "/host", "/run")).To(BeNil()) filesDest, err := fs.ReadDir(destDir) Expect(err).To(BeNil()) @@ -550,17 +552,17 @@ var _ = Describe("Utils", Label("utils"), func() { Expect(err).ShouldNot(HaveOccurred()) destDir, err := utils.TempDir(fs, "", "elementaltarget") Expect(err).ShouldNot(HaveOccurred()) - Expect(utils.SyncData(logger, fs, sourceDir, destDir)).To(BeNil()) + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) }) It("should NOT fail if destination does not exist", func() { sourceDir, err := os.MkdirTemp("", "elemental") err = os.WriteFile(filepath.Join(sourceDir, "testfile"), []byte("sdjfnsdjkfjkdsanfkjsnda"), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - err = utils.SyncData(logger, nil, sourceDir, "/welp") + err = utils.SyncData(logger, realRunner, nil, sourceDir, "/welp") Expect(err).To(BeNil()) }) It("should fail if source does not exist", func() { - Expect(utils.SyncData(logger, fs, "/welp", "/walp")).NotTo(BeNil()) + Expect(utils.SyncData(logger, realRunner, fs, "/welp", "/walp")).NotTo(BeNil()) }) }) Describe("IsLocalURI", Label("uri"), func() {