From f0fae82ad9f74331dfef206c293552e2bb17c891 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 18 Dec 2020 21:23:04 +0100 Subject: [PATCH] Add experimental zstd support Closes #97 --- cmd/build.go | 2 +- cmd/create-repo.go | 4 +- go.mod | 1 + pkg/compiler/artifact.go | 98 ++++++++++++++++++++++++- tests/integration/01_simple_zstd.sh | 108 ++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 5 deletions(-) create mode 100755 tests/integration/01_simple_zstd.sh diff --git a/cmd/build.go b/cmd/build.go index 93c1becd..78ad2026 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -313,7 +313,7 @@ func init() { buildCmd.Flags().String("values", "", "Build values file to interpolate with each package") buildCmd.Flags().String("destination", path, "Destination folder") - buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip") + buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd") buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image") buildCmd.Flags().Bool("push", false, "Push images to a hub") buildCmd.Flags().Bool("pull", false, "Pull images from a hub") diff --git a/cmd/create-repo.go b/cmd/create-repo.go index ed42b773..7e86efb7 100644 --- a/cmd/create-repo.go +++ b/cmd/create-repo.go @@ -159,9 +159,9 @@ func init() { createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.") createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.") - createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip") + createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd") createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename") - createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip") + createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd") createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename") RootCmd.AddCommand(createrepoCmd) diff --git a/go.mod b/go.mod index 385256bf..35843769 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/jedib0t/go-pretty/v6 v6.0.5 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 + github.com/klauspost/compress v1.8.3 github.com/klauspost/pgzip v1.2.1 github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d github.com/kyokomi/emoji v2.1.0+incompatible diff --git a/pkg/compiler/artifact.go b/pkg/compiler/artifact.go index 81eed3e3..f2d86193 100644 --- a/pkg/compiler/artifact.go +++ b/pkg/compiler/artifact.go @@ -28,6 +28,7 @@ import ( "regexp" system "github.com/docker/docker/pkg/system" + "github.com/klauspost/compress/zstd" gzip "github.com/klauspost/pgzip" //"strconv" @@ -47,8 +48,9 @@ import ( type CompressionImplementation string const ( - None CompressionImplementation = "none" // e.g. tar for standard packages - GZip CompressionImplementation = "gzip" + None CompressionImplementation = "none" // e.g. tar for standard packages + GZip CompressionImplementation = "gzip" + Zstandard CompressionImplementation = "zstd" ) type ArtifactIndex []Artifact @@ -242,6 +244,43 @@ func (a *PackageArtifact) SetPath(p string) { // Compress Archives and compress (TODO) to the artifact path func (a *PackageArtifact) Compress(src string, concurrency int) error { switch a.CompressionType { + + case Zstandard: + err := helpers.Tar(src, a.Path) + if err != nil { + return err + } + original, err := os.Open(a.Path) + if err != nil { + return err + } + defer original.Close() + + zstdFile := a.Path + ".zstd" + bufferedReader := bufio.NewReader(original) + + // Open a file for writing. + dst, err := os.Create(zstdFile) + if err != nil { + return err + } + + enc, err := zstd.NewWriter(dst) + if err != nil { + return err + } + _, err = io.Copy(enc, bufferedReader) + if err != nil { + enc.Close() + return err + } + if err := enc.Close(); err != nil { + return err + } + + os.RemoveAll(a.Path) // Remove original + a.Path = zstdFile + return nil case GZip: err := helpers.Tar(src, a.Path) if err != nil { @@ -367,6 +406,40 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc) switch a.CompressionType { + case Zstandard: + // Create the uncompressed archive + archive, err := os.Create(a.GetPath() + ".uncompressed") + if err != nil { + return err + } + defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer archive.Close() + + original, err := os.Open(a.Path) + if err != nil { + return errors.Wrap(err, "Cannot open "+a.Path) + } + defer original.Close() + + bufferedReader := bufio.NewReader(original) + + d, err := zstd.NewReader(bufferedReader) + if err != nil { + return err + } + defer d.Close() + + _, err = io.Copy(archive, d) + if err != nil { + return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed") + } + + err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst, + LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier) + if err != nil { + return err + } + return nil case GZip: // Create the uncompressed archive archive, err := os.Create(a.GetPath() + ".uncompressed") @@ -412,6 +485,27 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { func (a *PackageArtifact) FileList() ([]string, error) { var tr *tar.Reader switch a.CompressionType { + case Zstandard: + archive, err := os.Create(a.GetPath() + ".uncompressed") + if err != nil { + return []string{}, err + } + defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer archive.Close() + + original, err := os.Open(a.Path) + if err != nil { + return []string{}, errors.Wrap(err, "Cannot open "+a.Path) + } + defer original.Close() + + bufferedReader := bufio.NewReader(original) + r, err := zstd.NewReader(bufferedReader) + if err != nil { + return []string{}, err + } + defer r.Close() + tr = tar.NewReader(r) case GZip: // Create the uncompressed archive archive, err := os.Create(a.GetPath() + ".uncompressed") diff --git a/tests/integration/01_simple_zstd.sh b/tests/integration/01_simple_zstd.sh new file mode 100755 index 00000000..658fa8fb --- /dev/null +++ b/tests/integration/01_simple_zstd.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +export LUET_NOLOCK=true + +oneTimeSetUp() { +export tmpdir="$(mktemp -d)" +} + +oneTimeTearDown() { + rm -rf "$tmpdir" +} + +testBuild() { + mkdir $tmpdir/testbuild + luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null + buildst=$? + assertEquals 'builds successfully' "$buildst" "0" + assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zstd' ]" + assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zstd' ]" +} + +testRepo() { + assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]" + luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \ + --output $tmpdir/testbuild \ + --packages $tmpdir/testbuild \ + --name "test" \ + --descr "Test Repo" \ + --urls $tmpdir/testrootfs \ + --tree-compression zstd \ + --tree-filename foo.tar \ + --meta-filename repository.meta.tar \ + --meta-compression zstd \ + --type disk > /dev/null + + createst=$? + assertEquals 'create repo successfully' "$createst" "0" + assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]" + assertTrue 'create named tree in zstd' "[ -e '$tmpdir/testbuild/foo.tar.zstd' ]" + assertTrue 'create tree in zstd-only' "[ ! -e '$tmpdir/testbuild/foo.tar' ]" + assertTrue 'create named meta in zstd' "[ -e '$tmpdir/testbuild/repository.meta.tar.zstd' ]" + assertTrue 'create meta in zstd-only' "[ ! -e '$tmpdir/testbuild/repository.meta.tar' ]" +} + +testConfig() { + mkdir $tmpdir/testrootfs + cat < $tmpdir/luet.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "disk" + enable: true + urls: + - "$tmpdir/testbuild" +EOF + luet config --config $tmpdir/luet.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testInstall() { + luet install -y --config $tmpdir/luet.yaml test/c@1.0 + #luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]" +} + +testReInstall() { + output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0) + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertContains 'contains warning' "$output" 'No packages to install' +} + +testUnInstall() { + luet uninstall -y --config $tmpdir/luet.yaml test/c@1.0 + installst=$? + assertEquals 'uninstall test successfully' "$installst" "0" + assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]" +} + +testInstallAgain() { + assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]" + output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0) + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertNotContains 'contains warning' "$output" 'No packages to install' + assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]" + assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zstd' ]" +} + +testCleanup() { + luet cleanup --config $tmpdir/luet.yaml + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zstd' ]" +} + +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +