kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-08-31 14:52:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add experimental zstd support

Browse Source 
Closes #97
This commit is contained in:
Ettore Di Giacinto
2020-12-18 21:23:04 +01:00
parent 93f5f5d0b2
commit f0fae82ad9
5 changed files with 208 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/build.go
View File

@@ -313,7 +313,7 @@ func init() {
buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
buildCmd.Flags().String("destination", path, "Destination folder")
buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")
buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
buildCmd.Flags().Bool("push", false, "Push images to a hub")
buildCmd.Flags().Bool("pull", false, "Pull images from a hub")

4
cmd/create-repo.go
View File

@@ -159,9 +159,9 @@ func init() {
createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip")
createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip")
createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
RootCmd.AddCommand(createrepoCmd)

1
go.mod
View File

@@ -19,6 +19,7 @@ require (
github.com/jedib0t/go-pretty v4.3.0+incompatible
github.com/jedib0t/go-pretty/v6 v6.0.5
github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
github.com/klauspost/compress v1.8.3
github.com/klauspost/pgzip v1.2.1
github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
github.com/kyokomi/emoji v2.1.0+incompatible

98
pkg/compiler/artifact.go
View File

@@ -28,6 +28,7 @@ import (
"regexp"
system "github.com/docker/docker/pkg/system"
"github.com/klauspost/compress/zstd"
gzip "github.com/klauspost/pgzip"
//"strconv"
@@ -47,8 +48,9 @@ import (
type CompressionImplementation string
const (
None CompressionImplementation = "none" // e.g. tar for standard packages
GZip CompressionImplementation = "gzip"
None CompressionImplementation = "none" // e.g. tar for standard packages
GZip CompressionImplementation = "gzip"
Zstandard CompressionImplementation = "zstd"
)
type ArtifactIndex []Artifact
@@ -242,6 +244,43 @@ func (a *PackageArtifact) SetPath(p string) {
// Compress Archives and compress (TODO) to the artifact path
func (a *PackageArtifact) Compress(src string, concurrency int) error {
switch a.CompressionType {
case Zstandard:
err := helpers.Tar(src, a.Path)
if err != nil {
return err
}
original, err := os.Open(a.Path)
if err != nil {
return err
}
defer original.Close()
zstdFile := a.Path + ".zstd"
bufferedReader := bufio.NewReader(original)
// Open a file for writing.
dst, err := os.Create(zstdFile)
if err != nil {
return err
}
enc, err := zstd.NewWriter(dst)
if err != nil {
return err
}
_, err = io.Copy(enc, bufferedReader)
if err != nil {
enc.Close()
return err
}
if err := enc.Close(); err != nil {
return err
}
os.RemoveAll(a.Path) // Remove original
a.Path = zstdFile
return nil
case GZip:
err := helpers.Tar(src, a.Path)
if err != nil {
@@ -367,6 +406,40 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
switch a.CompressionType {
case Zstandard:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")
if err != nil {
return err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
if err != nil {
return errors.Wrap(err, "Cannot open "+a.Path)
}
defer original.Close()
bufferedReader := bufio.NewReader(original)
d, err := zstd.NewReader(bufferedReader)
if err != nil {
return err
}
defer d.Close()
_, err = io.Copy(archive, d)
if err != nil {
return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
}
err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
if err != nil {
return err
}
return nil
case GZip:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")
@@ -412,6 +485,27 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
func (a *PackageArtifact) FileList() ([]string, error) {
var tr *tar.Reader
switch a.CompressionType {
case Zstandard:
archive, err := os.Create(a.GetPath() + ".uncompressed")
if err != nil {
return []string{}, err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
if err != nil {
return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
}
defer original.Close()
bufferedReader := bufio.NewReader(original)
r, err := zstd.NewReader(bufferedReader)
if err != nil {
return []string{}, err
}
defer r.Close()
tr = tar.NewReader(r)
case GZip:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")

108
tests/integration/01_simple_zstd.sh Executable file
View File

@@ -0,0 +1,108 @@
#!/bin/bash
export LUET_NOLOCK=true
oneTimeSetUp() {
export tmpdir="$(mktemp -d)"
}
oneTimeTearDown() {
rm -rf "$tmpdir"
}
testBuild() {
mkdir $tmpdir/testbuild
luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null
buildst=$?
assertEquals 'builds successfully' "$buildst" "0"
assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zstd' ]"
assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zstd' ]"
}
testRepo() {
assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
--output $tmpdir/testbuild \
--packages $tmpdir/testbuild \
--name "test" \
--descr "Test Repo" \
--urls $tmpdir/testrootfs \
--tree-compression zstd \
--tree-filename foo.tar \
--meta-filename repository.meta.tar \
--meta-compression zstd \
--type disk > /dev/null
createst=$?
assertEquals 'create repo successfully' "$createst" "0"
assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
assertTrue 'create named tree in zstd' "[ -e '$tmpdir/testbuild/foo.tar.zstd' ]"
assertTrue 'create tree in zstd-only' "[ ! -e '$tmpdir/testbuild/foo.tar' ]"
assertTrue 'create named meta in zstd' "[ -e '$tmpdir/testbuild/repository.meta.tar.zstd' ]"
assertTrue 'create meta in zstd-only' "[ ! -e '$tmpdir/testbuild/repository.meta.tar' ]"
}
testConfig() {
mkdir $tmpdir/testrootfs
cat <<EOF > $tmpdir/luet.yaml
general:
debug: true
system:
rootfs: $tmpdir/testrootfs
database_path: "/"
database_engine: "boltdb"
config_from_host: true
repositories:
- name: "main"
type: "disk"
enable: true
urls:
- "$tmpdir/testbuild"
EOF
luet config --config $tmpdir/luet.yaml
res=$?
assertEquals 'config test successfully' "$res" "0"
}
testInstall() {
luet install -y --config $tmpdir/luet.yaml test/c@1.0
#luet install -y --config $tmpdir/luet.yaml test/c@1.0 > /dev/null
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
}
testReInstall() {
output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0)
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertContains 'contains warning' "$output" 'No packages to install'
}
testUnInstall() {
luet uninstall -y --config $tmpdir/luet.yaml test/c@1.0
installst=$?
assertEquals 'uninstall test successfully' "$installst" "0"
assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
}
testInstallAgain() {
assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
output=$(luet install -y --config $tmpdir/luet.yaml test/c@1.0)
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertNotContains 'contains warning' "$output" 'No packages to install'
assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zstd' ]"
}
testCleanup() {
luet cleanup --config $tmpdir/luet.yaml
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zstd' ]"
}
# Load shUnit2.
. "$ROOT_DIR/tests/integration/shunit2"/shunit2