diff --git a/pkg/api/core/config/config_protect.go b/pkg/api/core/config/config_protect.go index 42777acd..008658c0 100644 --- a/pkg/api/core/config/config_protect.go +++ b/pkg/api/core/config/config_protect.go @@ -57,14 +57,6 @@ func NewConfigProtect(annotationDir string) *ConfigProtect { } } -func (c *ConfigProtect) AddAnnotationDir(d string) { - c.AnnotationDir = d -} - -func (c *ConfigProtect) GetAnnotationDir() string { - return c.AnnotationDir -} - func (c *ConfigProtect) Map(files []string, protected []ConfigProtectConfFile) { for _, file := range files { diff --git a/pkg/api/core/image/delta_test.go b/pkg/api/core/image/delta_test.go index da430283..8a8362ca 100644 --- a/pkg/api/core/image/delta_test.go +++ b/pkg/api/core/image/delta_test.go @@ -72,6 +72,7 @@ var _ = Describe("Delta", func() { _, tmpdir, err := Extract( ctx, img2, + true, ExtractDeltaFiles(ctx, diff, []string{}, []string{}), ) Expect(err).ToNot(HaveOccurred()) @@ -87,6 +88,7 @@ var _ = Describe("Delta", func() { _, tmpdir, err := Extract( ctx, img2, + true, ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}), ) Expect(err).ToNot(HaveOccurred()) @@ -98,6 +100,7 @@ var _ = Describe("Delta", func() { _, tmpdir, err := Extract( ctx, img2, + true, ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}), ) Expect(err).ToNot(HaveOccurred()) @@ -110,6 +113,7 @@ var _ = Describe("Delta", func() { _, tmpdir, err := Extract( ctx, img2, + true, ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}), ) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/api/core/image/extract.go b/pkg/api/core/image/extract.go index e0db9fee..31d99f28 100644 --- a/pkg/api/core/image/extract.go +++ b/pkg/api/core/image/extract.go @@ -166,7 +166,7 @@ func ExtractFiles( } } -func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { +func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { defer reader.Close() perms := map[string][]int{} @@ -190,39 +190,41 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filt return 0, "", err } - for f, p := range perms { - ff := filepath.Join(output, f) - if _, err := os.Lstat(ff); err == nil { - if err := os.Lchown(ff, p[1], p[0]); err != nil { - ctx.Warning(err, "failed chowning file") + // TODO: Parametrize this + if keepPerms { + for f, p := range perms { + ff := filepath.Join(output, f) + if _, err := os.Lstat(ff); err == nil { + if err := os.Lchown(ff, p[1], p[0]); err != nil { + ctx.Warning(err, "failed chowning file") + } } } - } - for _, m := range []map[string]map[string]string{xattrs, paxrecords} { - for key, attrs := range m { - ff := filepath.Join(output, key) - for k, attr := range attrs { - if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil { - if errors.Is(err, syscall.ENOTSUP) { - ctx.Debug("ignored xattr %s in archive", key) + for _, m := range []map[string]map[string]string{xattrs, paxrecords} { + for key, attrs := range m { + ff := filepath.Join(output, key) + for k, attr := range attrs { + if err := system.Lsetxattr(ff, k, []byte(attr), 0); err != nil { + if errors.Is(err, syscall.ENOTSUP) { + ctx.Debug("ignored xattr %s in archive", key) + } } } } } } - return c, output, nil } -func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { +func Extract(ctx *types.Context, img v1.Image, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction") if err != nil { return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs") } - return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...) + return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, keepPerms, filter, opts...) } -func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { - return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...) +func ExtractTo(ctx *types.Context, img v1.Image, output string, keepPerms bool, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) { + return ExtractReader(ctx, mutate.Extract(img), output, keepPerms, filter, opts...) } diff --git a/pkg/api/core/image/extract_test.go b/pkg/api/core/image/extract_test.go index 4a4ebaec..f5427b80 100644 --- a/pkg/api/core/image/extract_test.go +++ b/pkg/api/core/image/extract_test.go @@ -58,6 +58,7 @@ var _ = Describe("Extract", func() { _, tmpdir, err := Extract( ctx, img, + true, ExtractFiles(ctx, "", []string{}, []string{}), ) Expect(err).ToNot(HaveOccurred()) @@ -71,6 +72,7 @@ var _ = Describe("Extract", func() { _, tmpdir, err := Extract( ctx, img, + true, ExtractFiles(ctx, "/usr", []string{}, []string{}), ) Expect(err).ToNot(HaveOccurred()) @@ -84,6 +86,7 @@ var _ = Describe("Extract", func() { _, tmpdir, err := Extract( ctx, img, + true, ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}), ) Expect(err).ToNot(HaveOccurred()) @@ -98,6 +101,7 @@ var _ = Describe("Extract", func() { _, tmpdir, err := Extract( ctx, img, + true, ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}), ) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/api/core/types/artifact/artifact.go b/pkg/api/core/types/artifact/artifact.go index de6c9a76..b2190c5a 100644 --- a/pkg/api/core/types/artifact/artifact.go +++ b/pkg/api/core/types/artifact/artifact.go @@ -28,6 +28,7 @@ import ( "path" "path/filepath" + "github.com/docker/docker/pkg/pools" v1 "github.com/google/go-containerregistry/pkg/v1" zstd "github.com/klauspost/compress/zstd" gzip "github.com/klauspost/pgzip" @@ -35,6 +36,7 @@ import ( //"strconv" "strings" + containerdCompression "github.com/containerd/containerd/archive/compression" config "github.com/mudler/luet/pkg/api/core/config" "github.com/mudler/luet/pkg/api/core/image" types "github.com/mudler/luet/pkg/api/core/types" @@ -67,8 +69,8 @@ type PackageArtifact struct { Runtime *pkg.DefaultPackage `json:"runtime,omitempty"` } -func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) { - _, tmpdiffs, err := image.Extract(ctx, img, filter) +func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, keepPerms bool, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) { + _, tmpdiffs, err := image.Extract(ctx, img, keepPerms, filter) if err != nil { return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") } @@ -385,6 +387,85 @@ func hashFileContent(path string) (string, error) { return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil } +func replaceFileTarWrapper(dst string, inputTarStream io.ReadCloser, mods []string, fn func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)) io.ReadCloser { + pipeReader, pipeWriter := io.Pipe() + + go func() { + tarReader := tar.NewReader(inputTarStream) + tarWriter := tar.NewWriter(pipeWriter) + defer inputTarStream.Close() + defer tarWriter.Close() + + modify := func(name string, original *tar.Header, tarReader io.Reader) error { + header, data, err := fn(dst, name, original, tarReader) + switch { + case err != nil: + return err + case header == nil: + return nil + } + + if header.Name == "" { + header.Name = name + } + header.Size = int64(len(data)) + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if len(data) != 0 { + if _, err := tarWriter.Write(data); err != nil { + return err + } + } + return nil + } + var remaining []string + var err error + var originalHeader *tar.Header + for { + originalHeader, err = tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + pipeWriter.CloseWithError(err) + return + } + + if helpers.Contains(mods, originalHeader.Name) { + // No modifiers for this file, copy the header and data + if err := tarWriter.WriteHeader(originalHeader); err != nil { + pipeWriter.CloseWithError(err) + return + } + if _, err := pools.Copy(tarWriter, tarReader); err != nil { + pipeWriter.CloseWithError(err) + return + } + remaining = append(remaining, originalHeader.Name) + continue + } + + if err := modify(originalHeader.Name, originalHeader, tarReader); err != nil { + pipeWriter.CloseWithError(err) + return + } + } + + // Apply the modifiers that haven't matched any files in the archive + for _, name := range remaining { + if err := modify(name, nil, nil); err != nil { + pipeWriter.CloseWithError(err) + return + } + } + + pipeWriter.Close() + + }() + return pipeReader +} + func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) { return func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) { // If the destination path already exists I rename target file name with postfix. @@ -448,8 +529,7 @@ func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *t } } -func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string { - ans := []string{} +func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) (res []string) { annotationDir := "" if !ctx.Config.ConfigProtectSkip { @@ -468,159 +548,75 @@ func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string { cp.Map(a.Files, ctx.Config.GetConfigProtectConfFiles()) // NOTE: for unpack we need files path without initial / - ans = cp.GetProtectFiles(false) + res = cp.GetProtectFiles(false) } - return ans + return } // Unpack Untar and decompress (TODO) to the given path func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool) error { - if !strings.HasPrefix(dst, "/") { + + if !strings.HasPrefix(dst, string(os.PathSeparator)) { return errors.New("destination must be an absolute path") } // Create protectedFiles := a.GetProtectFiles(ctx) - tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc(ctx)) + mod := tarModifierWrapperFunc(ctx) + //tarModifier := helpers.NewTarModifierWrapper(dst, mod) - switch a.CompressionType { - case compression.Zstandard: - // Create the uncompressed archive - archive, err := os.Create(a.Path + ".uncompressed") - if err != nil { - return err - } - defer os.RemoveAll(a.Path + ".uncompressed") - defer archive.Close() - - original, err := os.Open(a.Path) - if err != nil { - return errors.Wrap(err, "Cannot open "+a.Path) - } - defer original.Close() - - bufferedReader := bufio.NewReader(original) - - d, err := zstd.NewReader(bufferedReader) - if err != nil { - return err - } - defer d.Close() - - _, err = io.Copy(archive, d) - if err != nil { - return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed") - } - - err = helpers.UntarProtect(a.Path+".uncompressed", dst, - ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier) - if err != nil { - return err - } - return nil - case compression.GZip: - // Create the uncompressed archive - archive, err := os.Create(a.Path + ".uncompressed") - if err != nil { - return err - } - defer os.RemoveAll(a.Path + ".uncompressed") - defer archive.Close() - - original, err := os.Open(a.Path) - if err != nil { - return errors.Wrap(err, "Cannot open "+a.Path) - } - defer original.Close() - - bufferedReader := bufio.NewReader(original) - r, err := gzip.NewReader(bufferedReader) - if err != nil { - return err - } - defer r.Close() - - _, err = io.Copy(archive, r) - if err != nil { - return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed") - } - - err = helpers.UntarProtect(a.Path+".uncompressed", dst, - ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier) - if err != nil { - return err - } - return nil - // Defaults to tar only (covers when "none" is supplied) - default: - return helpers.UntarProtect(a.Path, dst, ctx.Config.GetGeneral().SameOwner, - protectedFiles, tarModifier) + archiveFile, err := os.Open(a.Path) + if err != nil { + return errors.Wrap(err, "Cannot open "+a.Path) } - return errors.New("Compression type must be supplied") + defer archiveFile.Close() + + decompressed, err := containerdCompression.DecompressStream(archiveFile) + if err != nil { + return errors.Wrap(err, "Cannot open "+a.Path) + } + defer decompressed.Close() + + replacerArchive := replaceFileTarWrapper(dst, decompressed, protectedFiles, mod) + defer replacerArchive.Close() + + // or with filter? + // func(header *tar.Header) (bool, error) { + // if helpers.Contains(protectedFiles, header.Name) { + // newHead, _, err := mod(dst, header.Name, header, decompressed) + // if err != nil { + // return false, err + // } + // header.Name = newHead.Name + // // Override target path + // //target = filepath.Join(dest, header.Name) + // } + // // tarModifier.Modifier() + // return true, nil + // }, + _, _, err = image.ExtractReader(ctx, replacerArchive, dst, ctx.Config.GetGeneral().SameOwner, nil) + return err } // FileList generates the list of file of a package from the local archive func (a *PackageArtifact) FileList() ([]string, error) { - var tr *tar.Reader - switch a.CompressionType { - case compression.Zstandard: - archive, err := os.Create(a.Path + ".uncompressed") - if err != nil { - return []string{}, err - } - defer os.RemoveAll(a.Path + ".uncompressed") - defer archive.Close() - - original, err := os.Open(a.Path) - if err != nil { - return []string{}, errors.Wrap(err, "Cannot open "+a.Path) - } - defer original.Close() - - bufferedReader := bufio.NewReader(original) - r, err := zstd.NewReader(bufferedReader) - if err != nil { - return []string{}, err - } - defer r.Close() - tr = tar.NewReader(r) - case compression.GZip: - // Create the uncompressed archive - archive, err := os.Create(a.Path + ".uncompressed") - if err != nil { - return []string{}, err - } - defer os.RemoveAll(a.Path + ".uncompressed") - defer archive.Close() - - original, err := os.Open(a.Path) - if err != nil { - return []string{}, errors.Wrap(err, "Cannot open "+a.Path) - } - defer original.Close() - - bufferedReader := bufio.NewReader(original) - r, err := gzip.NewReader(bufferedReader) - if err != nil { - return []string{}, err - } - defer r.Close() - tr = tar.NewReader(r) - - // Defaults to tar only (covers when "none" is supplied) - default: - tarFile, err := os.Open(a.Path) - if err != nil { - return []string{}, errors.Wrap(err, "Could not open package archive") - } - defer tarFile.Close() - tr = tar.NewReader(tarFile) - - } - var files []string + + archiveFile, err := os.Open(a.Path) + if err != nil { + return files, errors.Wrap(err, "Cannot open "+a.Path) + } + defer archiveFile.Close() + + decompressed, err := containerdCompression.DecompressStream(archiveFile) + if err != nil { + return files, errors.Wrap(err, "Cannot open "+a.Path) + } + defer decompressed.Close() + tr := tar.NewReader(decompressed) + // untar each segment for { hdr, err := tr.Next() diff --git a/pkg/api/core/types/artifact/artifact_test.go b/pkg/api/core/types/artifact/artifact_test.go index c012f9b5..bd886b9f 100644 --- a/pkg/api/core/types/artifact/artifact_test.go +++ b/pkg/api/core/types/artifact/artifact_test.go @@ -162,6 +162,7 @@ RUN echo bar > /test2`)) ctx, img, result, + false, nil, ) Expect(err).ToNot(HaveOccurred()) @@ -211,6 +212,7 @@ RUN echo bar > /test2`)) ctx, img, result, + false, nil, ) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index af631fd2..b7acae2c 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -248,6 +248,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi _, rootfs, err := image.Extract( cs.Options.Context, img, + keepPermissions, image.ExtractFiles( cs.Options.Context, p.GetPackageDir(), @@ -316,6 +317,7 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co ref2, cs.Options.CompressionType, p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")), + keepPermissions, image.ExtractDeltaFiles(cs.Options.Context, diff, p.GetIncludes(), p.GetExcludes()), ) if err != nil { diff --git a/pkg/helpers/docker/docker.go b/pkg/helpers/docker/docker.go index 1203a33c..3f0e930a 100644 --- a/pkg/helpers/docker/docker.go +++ b/pkg/helpers/docker/docker.go @@ -213,12 +213,8 @@ func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, a ctx, img, dest, - luetimages.ExtractFiles( - ctx, - "", - []string{}, - []string{}, - ), + true, + nil, ) if err != nil { return nil, err diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go index 906a8acc..a897f4c5 100644 --- a/pkg/installer/installer.go +++ b/pkg/installer/installer.go @@ -348,6 +348,7 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock defer wg.Done() for p := range c { + if p.Uninstall.Package != nil { l.Options.Context.Debug("Replacing package inplace") toUninstall, uninstall, err := l.generateUninstallFn(p.Uninstall.Option, s, p.Uninstall.Package) @@ -356,8 +357,10 @@ func (l *LuetInstaller) installerOpWorker(i int, wg *sync.WaitGroup, systemLock continue //return errors.Wrap(err, "while computing uninstall") } - + systemLock.Lock() err = uninstall() + systemLock.Unlock() + if err != nil { l.Options.Context.Error("Failed uninstall for ", packsToList(toUninstall)) continue diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index b5be99d8..45565764 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -929,7 +929,11 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem } ctx.Debug("Decompress tree of the repository " + r.Name + "...") - err = treeFileArtifact.Unpack(ctx, treefs, true) + if _, err := os.Lstat(treefs); os.IsNotExist(err) { + os.MkdirAll(treefs, 0600) + } + + err = treeFileArtifact.Unpack(ctx, treefs, false) if err != nil { return nil, errors.Wrap(err, "Error met while unpacking tree") } @@ -937,7 +941,7 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem // FIXME: It seems that tar with only one file doesn't create destination // directory. I create directory directly for now. os.MkdirAll(metafs, os.ModePerm) - err = metaFileArtifact.Unpack(ctx, metafs, true) + err = metaFileArtifact.Unpack(ctx, metafs, false) if err != nil { return nil, errors.Wrap(err, "Error met while unpacking metadata") } diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 2bc807e3..0e411e72 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -204,6 +204,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi d.context, img, repoTemp, + d.context.Config.GetGeneral().SameOwner, nil, ) if err != nil {