diff --git a/pkg/api/core/image/delta.go b/pkg/api/core/image/delta.go new file mode 100644 index 00000000..4e4667ae --- /dev/null +++ b/pkg/api/core/image/delta.go @@ -0,0 +1,111 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image + +import ( + "archive/tar" + "io" + "regexp" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" +) + +func compileRegexes(regexes []string) []*regexp.Regexp { + var result []*regexp.Regexp + for _, i := range regexes { + r, e := regexp.Compile(i) + if e != nil { + continue + } + result = append(result, r) + } + return result +} + +type ImageDiffNode struct { + Name string `json:"Name"` + Size int `json:"Size"` +} +type ImageDiff struct { + Additions []ImageDiffNode `json:"Adds"` + Deletions []ImageDiffNode `json:"Dels"` + Changes []ImageDiffNode `json:"Mods"` +} + +func Delta(srcimg, dstimg v1.Image) (res ImageDiff, err error) { + srcReader := mutate.Extract(srcimg) + defer srcReader.Close() + + dstReader := mutate.Extract(dstimg) + defer dstReader.Close() + + filesSrc, filesDst := map[string]int64{}, map[string]int64{} + + srcTar := tar.NewReader(srcReader) + dstTar := tar.NewReader(dstReader) + + for { + var hdr *tar.Header + hdr, err = srcTar.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return + } + filesSrc[hdr.Name] = hdr.Size + } + + for { + var hdr *tar.Header + hdr, err = dstTar.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return + } + filesDst[hdr.Name] = hdr.Size + } + err = nil + + for f, size := range filesDst { + if size2, exist := filesSrc[f]; exist && size2 != size { + res.Changes = append(res.Changes, ImageDiffNode{ + Name: f, + Size: int(size), + }) + } else if !exist { + res.Additions = append(res.Additions, ImageDiffNode{ + Name: f, + Size: int(size), + }) + } + } + for f, size := range filesSrc { + if _, exist := filesDst[f]; !exist { + res.Deletions = append(res.Deletions, ImageDiffNode{ + Name: f, + Size: int(size), + }) + } + } + + return +} diff --git a/pkg/api/core/image/delta_test.go b/pkg/api/core/image/delta_test.go new file mode 100644 index 00000000..877cc666 --- /dev/null +++ b/pkg/api/core/image/delta_test.go @@ -0,0 +1,136 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + daemon "github.com/google/go-containerregistry/pkg/v1/daemon" + . "github.com/mudler/luet/pkg/api/core/image" + "github.com/mudler/luet/pkg/api/core/types" + "github.com/mudler/luet/pkg/helpers/file" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("DeltaInfo", func() { + Context("Generates deltas of images", func() { + It("Correctly detect packages", func() { + ref, err := name.ParseReference("alpine") + Expect(err).ToNot(HaveOccurred()) + + img, err := daemon.Image(ref) + Expect(err).ToNot(HaveOccurred()) + + layers, err := Delta(img, img) + Expect(err).ToNot(HaveOccurred()) + Expect(len(layers.Changes)).To(Equal(0)) + Expect(len(layers.Additions)).To(Equal(0)) + Expect(len(layers.Deletions)).To(Equal(0)) + }) + + Context("ExtractDeltaFiles", func() { + ctx := types.NewContext() + var tmpfile *os.File + var ref, ref2 name.Reference + var img, img2 v1.Image + var diff ImageDiff + var err error + + BeforeEach(func() { + ctx = types.NewContext() + + tmpfile, err = ioutil.TempFile("", "delta") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpfile.Name()) // clean up + + ref, err = name.ParseReference("alpine") + Expect(err).ToNot(HaveOccurred()) + + ref2, err = name.ParseReference("golang:alpine") + Expect(err).ToNot(HaveOccurred()) + + img, err = daemon.Image(ref) + Expect(err).ToNot(HaveOccurred()) + img2, err = daemon.Image(ref2) + Expect(err).ToNot(HaveOccurred()) + + diff, err = Delta(img, img2) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(diff.Additions) > 0).To(BeTrue()) + Expect(len(diff.Changes) > 0).To(BeTrue()) + Expect(len(diff.Deletions) > 0).To(BeTrue()) + }) + + It("Extract all deltas", func() { + tmpdir, err := Extract( + ctx, + img2, + ExtractDeltaFiles(ctx, diff, []string{}, []string{}), + ) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + fmt.Println(file.ListDir(tmpdir)) + Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue()) + Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse()) + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue()) + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue()) + }) + + It("Extract deltas and excludes stuff", func() { + tmpdir, err := Extract( + ctx, + img2, + ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}), + ) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse()) + }) + It("Extract deltas and excludes stuff", func() { + tmpdir, err := Extract( + ctx, + img2, + ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}), + ) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue()) + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse()) + }) + + It("Extract deltas and excludes stuff", func() { + tmpdir, err := Extract( + ctx, + img2, + ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}), + ) + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue()) + Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeFalse()) + }) + }) + }) +}) diff --git a/pkg/api/core/image/extract.go b/pkg/api/core/image/extract.go new file mode 100644 index 00000000..d090cee7 --- /dev/null +++ b/pkg/api/core/image/extract.go @@ -0,0 +1,119 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image + +import ( + "archive/tar" + "context" + + containerdarchive "github.com/containerd/containerd/archive" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/mudler/luet/pkg/api/core/types" + "github.com/pkg/errors" +) + +// Extract dir: +// -> First extract delta considering the dir +// Afterward create artifact pointing to the dir + +// ExtractDeltaFiles returns an handler to extract files in a list +func ExtractDeltaFiles( + ctx *types.Context, + d ImageDiff, + includes []string, excludes []string, +) func(h *tar.Header) (bool, error) { + + includeRegexp := compileRegexes(includes) + excludeRegexp := compileRegexes(excludes) + + return func(h *tar.Header) (bool, error) { + switch { + case len(includes) == 0 && len(excludes) != 0: + for _, a := range d.Additions { + if h.Name == a.Name { + for _, i := range excludeRegexp { + if i.MatchString(a.Name) && h.Name == a.Name { + return false, nil + } + } + ctx.Info("Adding name", h.Name) + + return true, nil + } + } + return false, nil + case len(includes) > 0 && len(excludes) == 0: + for _, a := range d.Additions { + for _, i := range includeRegexp { + if i.MatchString(a.Name) && h.Name == a.Name { + ctx.Info("Adding name", h.Name) + + return true, nil + } + } + } + return false, nil + case len(includes) != 0 && len(excludes) != 0: + for _, a := range d.Additions { + for _, i := range includeRegexp { + if i.MatchString(a.Name) && h.Name == a.Name { + for _, e := range excludeRegexp { + if e.MatchString(a.Name) { + return false, nil + } + } + ctx.Info("Adding name", h.Name) + + return true, nil + } + } + } + return false, nil + default: + for _, a := range d.Additions { + if h.Name == a.Name { + ctx.Info("Adding name", h.Name) + + return true, nil + } + } + return false, nil + } + + } +} + +func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) { + src := mutate.Extract(img) + defer src.Close() + + tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction") + if err != nil { + return "", errors.Wrap(err, "Error met while creating tempdir for rootfs") + } + + if filter != nil { + opts = append(opts, containerdarchive.WithFilter(filter)) + } + + _, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...) + if err != nil { + return "", err + } + + return tmpdiffs, nil +} diff --git a/pkg/api/core/image/mutator_suite_test.go b/pkg/api/core/image/mutator_suite_test.go new file mode 100644 index 00000000..5f8e6a7d --- /dev/null +++ b/pkg/api/core/image/mutator_suite_test.go @@ -0,0 +1,28 @@ +// Copyright © 2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package image_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestMutator(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Mutator API Suite") +} diff --git a/pkg/api/core/types/artifact/artifact.go b/pkg/api/core/types/artifact/artifact.go index 682090b1..2d7cbe17 100644 --- a/pkg/api/core/types/artifact/artifact.go +++ b/pkg/api/core/types/artifact/artifact.go @@ -27,16 +27,16 @@ import ( "os" "path" "path/filepath" - "regexp" + v1 "github.com/google/go-containerregistry/pkg/v1" zstd "github.com/klauspost/compress/zstd" gzip "github.com/klauspost/pgzip" //"strconv" "strings" - "sync" config "github.com/mudler/luet/pkg/api/core/config" + "github.com/mudler/luet/pkg/api/core/image" types "github.com/mudler/luet/pkg/api/core/types" bus "github.com/mudler/luet/pkg/bus" backend "github.com/mudler/luet/pkg/compiler/backend" @@ -67,6 +67,22 @@ type PackageArtifact struct { Runtime *pkg.DefaultPackage `json:"runtime,omitempty"` } +func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) { + tmpdiffs, err := image.Extract(ctx, img, filter) + if err != nil { + return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") + } + defer os.RemoveAll(tmpdiffs) // clean up + + a := NewPackageArtifact(output) + a.CompressionType = t + err = a.Compress(tmpdiffs, 1) + if err != nil { + return nil, errors.Wrap(err, "Error met while creating package archive") + } + return a, nil +} + func (p *PackageArtifact) ShallowCopy() *PackageArtifact { copy := *p return © @@ -626,181 +642,3 @@ func (a *PackageArtifact) FileList() ([]string, error) { } return files, nil } - -type CopyJob struct { - Src, Dst string - Artifact string -} - -func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) { - defer wg.Done() - - for job := range s { - _, err := os.Lstat(job.Dst) - if err != nil { - ctx.Debug("Copying ", job.Src) - if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil { - ctx.Warning("Error copying", job, err) - } - } - } -} - -func compileRegexes(regexes []string) []*regexp.Regexp { - var result []*regexp.Regexp - for _, i := range regexes { - r, e := regexp.Compile(i) - if e != nil { - continue - } - result = append(result, r) - } - return result -} - -type ArtifactNode struct { - Name string `json:"Name"` - Size int `json:"Size"` -} -type ArtifactDiffs struct { - Additions []ArtifactNode `json:"Adds"` - Deletions []ArtifactNode `json:"Dels"` - Changes []ArtifactNode `json:"Mods"` -} - -type ArtifactLayer struct { - FromImage string `json:"Image1"` - ToImage string `json:"Image2"` - Diffs ArtifactDiffs `json:"Diff"` -} - -// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format -func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) { - - archive, err := ctx.Config.GetSystem().TempDir("archive") - if err != nil { - return nil, errors.Wrap(err, "Error met while creating tempdir for archive") - } - defer os.RemoveAll(archive) // clean up - - if strings.HasSuffix(src, ".tar") { - rootfs, err := ctx.Config.GetSystem().TempDir("rootfs") - if err != nil { - return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") - } - defer os.RemoveAll(rootfs) // clean up - err = helpers.Untar(src, rootfs, keepPerms) - if err != nil { - return nil, errors.Wrap(err, "Error met while unpacking rootfs") - } - src = rootfs - } - - toCopy := make(chan CopyJob) - - var wg = new(sync.WaitGroup) - for i := 0; i < concurrency; i++ { - wg.Add(1) - go worker(ctx, i, wg, toCopy) - } - - // Handle includes in spec. If specified they filter what gets in the package - - if len(includes) > 0 && len(excludes) == 0 { - includeRegexp := compileRegexes(includes) - for _, l := range layers { - // Consider d.Additions (and d.Changes? - warn at least) only - ADDS: - for _, a := range l.Diffs.Additions { - for _, i := range includeRegexp { - if i.MatchString(a.Name) { - toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name} - continue ADDS - } - } - } - for _, a := range l.Diffs.Changes { - ctx.Debug("File ", a.Name, " changed") - } - for _, a := range l.Diffs.Deletions { - ctx.Debug("File ", a.Name, " deleted") - } - } - - } else if len(includes) == 0 && len(excludes) != 0 { - excludeRegexp := compileRegexes(excludes) - for _, l := range layers { - // Consider d.Additions (and d.Changes? - warn at least) only - ADD: - for _, a := range l.Diffs.Additions { - for _, i := range excludeRegexp { - if i.MatchString(a.Name) { - continue ADD - } - } - toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name} - } - for _, a := range l.Diffs.Changes { - ctx.Debug("File ", a.Name, " changed") - } - for _, a := range l.Diffs.Deletions { - ctx.Debug("File ", a.Name, " deleted") - } - } - - } else if len(includes) != 0 && len(excludes) != 0 { - includeRegexp := compileRegexes(includes) - excludeRegexp := compileRegexes(excludes) - - for _, l := range layers { - // Consider d.Additions (and d.Changes? - warn at least) only - EXCLUDES: - for _, a := range l.Diffs.Additions { - for _, i := range includeRegexp { - if i.MatchString(a.Name) { - for _, e := range excludeRegexp { - if e.MatchString(a.Name) { - continue EXCLUDES - } - } - toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name} - continue EXCLUDES - } - } - } - for _, a := range l.Diffs.Changes { - ctx.Debug("File ", a.Name, " changed") - } - for _, a := range l.Diffs.Deletions { - ctx.Debug("File ", a.Name, " deleted") - } - } - - } else { - // Otherwise just grab all - for _, l := range layers { - // Consider d.Additions (and d.Changes? - warn at least) only - for _, a := range l.Diffs.Additions { - ctx.Debug("File ", a.Name, " added") - toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name} - } - for _, a := range l.Diffs.Changes { - ctx.Debug("File ", a.Name, " changed") - } - for _, a := range l.Diffs.Deletions { - ctx.Debug("File ", a.Name, " deleted") - } - } - } - - close(toCopy) - wg.Wait() - - a := NewPackageArtifact(dst) - a.CompressionType = t - err = a.Compress(archive, concurrency) - if err != nil { - return nil, errors.Wrap(err, "Error met while creating package archive") - } - return a, nil -} diff --git a/pkg/api/core/types/artifact/artifact_test.go b/pkg/api/core/types/artifact/artifact_test.go index 14592077..acc0afc4 100644 --- a/pkg/api/core/types/artifact/artifact_test.go +++ b/pkg/api/core/types/artifact/artifact_test.go @@ -22,7 +22,6 @@ import ( "github.com/mudler/luet/pkg/api/core/types" . "github.com/mudler/luet/pkg/api/core/types/artifact" - "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler/backend" backend "github.com/mudler/luet/pkg/compiler/backend" compression "github.com/mudler/luet/pkg/compiler/types/compression" @@ -30,7 +29,6 @@ import ( compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" . "github.com/mudler/luet/pkg/compiler" - helpers "github.com/mudler/luet/pkg/helpers" fileHelper "github.com/mudler/luet/pkg/helpers/file" pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/tree" @@ -117,53 +115,7 @@ RUN echo bar > /test2`)) } Expect(b.BuildImage(opts2)).ToNot(HaveOccurred()) Expect(b.ExportImage(opts2)).ToNot(HaveOccurred()) - Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue()) - diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2) - Expect(err).ToNot(HaveOccurred()) - artifacts := []ArtifactNode{{ - Name: "/luetbuild/LuetDockerfile", - Size: 175, - }} - if os.Getenv("DOCKER_BUILDKIT") == "1" { - artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0}) - } - artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4}) - artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4}) - - Expect(diffs).To(Equal( - []ArtifactLayer{{ - FromImage: "luet/base", - ToImage: "test", - Diffs: ArtifactDiffs{ - Additions: artifacts, - }, - }})) - err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false) - Expect(err).ToNot(HaveOccurred()) - - a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None) - Expect(err).ToNot(HaveOccurred()) - Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue()) - err = helpers.Untar(a.Path, unpacked, false) - Expect(err).ToNot(HaveOccurred()) - Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue()) - Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue()) - content1, err := fileHelper.Read(filepath.Join(unpacked, "test")) - Expect(err).ToNot(HaveOccurred()) - Expect(content1).To(Equal("foo\n")) - content2, err := fileHelper.Read(filepath.Join(unpacked, "test2")) - Expect(err).ToNot(HaveOccurred()) - Expect(content2).To(Equal("bar\n")) - - err = a.Hash() - Expect(err).ToNot(HaveOccurred()) - err = a.Verify() - Expect(err).ToNot(HaveOccurred()) - Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred()) - - err = a.Verify() - Expect(err).To(HaveOccurred()) }) It("Generates packages images", func() { diff --git a/pkg/compiler/backend.go b/pkg/compiler/backend.go index d6d091b7..b6ff119b 100644 --- a/pkg/compiler/backend.go +++ b/pkg/compiler/backend.go @@ -1,14 +1,8 @@ package compiler import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/mudler/luet/pkg/api/core/types" - artifact "github.com/mudler/luet/pkg/api/core/types/artifact" "github.com/mudler/luet/pkg/compiler/backend" "github.com/pkg/errors" @@ -42,204 +36,6 @@ type CompilerBackend interface { Push(opts backend.Options) error ImageAvailable(string) bool + ImageReference(img1 string) (v1.Image, error) ImageExists(string) bool } - -// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods -// example of json return: [ -// { -// "Image1": "luet/base", -// "Image2": "alpine", -// "DiffType": "File", -// "Diff": { -// "Adds": null, -// "Dels": [ -// { -// "Name": "/luetbuild", -// "Size": 5830706 -// }, -// { -// "Name": "/luetbuild/Dockerfile", -// "Size": 50 -// }, -// { -// "Name": "/luetbuild/output1", -// "Size": 5830656 -// } -// ], -// "Mods": null -// } -// } -// ] -func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) { - - res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName} - - tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction") - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") - } - defer os.RemoveAll(tmpdiffs) // clean up - - srcRootFS, err := ioutil.TempDir(tmpdiffs, "src") - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") - } - defer os.RemoveAll(srcRootFS) // clean up - - dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst") - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") - } - defer os.RemoveAll(dstRootFS) // clean up - - srcImageExtract := backend.Options{ - ImageName: fromImage.ImageName, - Destination: srcRootFS, - } - ctx.Debug("Extracting source image", fromImage.ImageName) - err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName) - } - - dstImageExtract := backend.Options{ - ImageName: toImage.ImageName, - Destination: dstRootFS, - } - ctx.Debug("Extracting destination image", toImage.ImageName) - err = b.ExtractRootfs(dstImageExtract, false) - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName) - } - - // Get Additions/Changes. dst -> src - err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - - realpath := strings.Replace(path, dstRootFS, "", -1) - fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath)) - if err == nil { - var sizeA, sizeB int64 - sizeA = fileInfo.Size() - - if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil { - sizeB = s.Size() - } - - if sizeA != sizeB { - // fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath)) - res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{ - Name: filepath.Join("/", realpath), - Size: int(sizeB), - }) - } else { - // fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath)) - } - } else { - var sizeB int64 - - if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil { - sizeB = s.Size() - } - res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{ - Name: filepath.Join("/", realpath), - Size: int(sizeB), - }) - - // fmt.Println("File created", path, filepath.Join(srcRootFS, realpath)) - } - - return nil - }) - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination") - } - - // Get deletions. src -> dst - err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - - realpath := strings.Replace(path, srcRootFS, "", -1) - if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil { - // fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath)) - res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{ - Name: filepath.Join("/", realpath), - }) - } - - return nil - }) - if err != nil { - return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source") - } - - diffs := []artifact.ArtifactLayer{res} - - if ctx.Config.GetGeneral().Debug { - summary := ComputeArtifactLayerSummary(diffs) - for _, l := range summary.Layers { - ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)", - l.FromImage, l.ToImage, - l.AddFiles, l.AddSizes, - l.DelFiles, l.DelSizes, - l.ChangeFiles, l.ChangeSizes)) - } - } - - return diffs, nil -} - -type ArtifactLayerSummary struct { - FromImage string `json:"image1"` - ToImage string `json:"image2"` - AddFiles int `json:"add_files"` - AddSizes int64 `json:"add_sizes"` - DelFiles int `json:"del_files"` - DelSizes int64 `json:"del_sizes"` - ChangeFiles int `json:"change_files"` - ChangeSizes int64 `json:"change_sizes"` -} - -type ArtifactLayersSummary struct { - Layers []ArtifactLayerSummary `json:"summary"` -} - -func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary { - - ans := ArtifactLayersSummary{ - Layers: make([]ArtifactLayerSummary, 0), - } - - for _, layer := range diffs { - sum := ArtifactLayerSummary{ - FromImage: layer.FromImage, - ToImage: layer.ToImage, - AddFiles: 0, - AddSizes: 0, - DelFiles: 0, - DelSizes: 0, - ChangeFiles: 0, - ChangeSizes: 0, - } - for _, a := range layer.Diffs.Additions { - sum.AddFiles++ - sum.AddSizes += int64(a.Size) - } - for _, d := range layer.Diffs.Deletions { - sum.DelFiles++ - sum.DelSizes += int64(d.Size) - } - for _, c := range layer.Diffs.Changes { - sum.ChangeFiles++ - sum.ChangeSizes += int64(c.Size) - } - ans.Layers = append(ans.Layers, sum) - } - - return ans -} diff --git a/pkg/compiler/backend/simpledocker.go b/pkg/compiler/backend/simpledocker.go index fa8d8f02..57f4c268 100644 --- a/pkg/compiler/backend/simpledocker.go +++ b/pkg/compiler/backend/simpledocker.go @@ -23,10 +23,13 @@ import ( "path/filepath" "strings" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/mudler/luet/pkg/api/core/types" bus "github.com/mudler/luet/pkg/bus" fileHelper "github.com/mudler/luet/pkg/helpers/file" + v1 "github.com/google/go-containerregistry/pkg/v1" capi "github.com/mudler/docker-companion/api" "github.com/mudler/luet/pkg/helpers" @@ -144,6 +147,19 @@ func (s *SimpleDocker) Push(opts Options) error { return nil } +func (s *SimpleDocker) ImageReference(a string) (v1.Image, error) { + ref, err := name.ParseReference(a) + if err != nil { + return nil, err + } + + img, err := daemon.Image(ref) + if err != nil { + return nil, err + } + return img, nil +} + func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error { if err := s.BuildImage(opts); err != nil { return errors.Wrap(err, "Failed building image") diff --git a/pkg/compiler/backend/simpledocker_test.go b/pkg/compiler/backend/simpledocker_test.go index 8411fc91..61b36325 100644 --- a/pkg/compiler/backend/simpledocker_test.go +++ b/pkg/compiler/backend/simpledocker_test.go @@ -17,8 +17,6 @@ package backend_test import ( "github.com/mudler/luet/pkg/api/core/types" - "github.com/mudler/luet/pkg/api/core/types/artifact" - "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/compiler/backend" . "github.com/mudler/luet/pkg/compiler/backend" @@ -107,35 +105,6 @@ RUN echo bar > /test2`)) Expect(b.ExportImage(opts2)).ToNot(HaveOccurred()) Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue()) - artifacts := []artifact.ArtifactNode{{ - Name: "/luetbuild/LuetDockerfile", - Size: 175, - }} - if os.Getenv("DOCKER_BUILDKIT") == "1" { - artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0}) - } - artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4}) - artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4}) - - Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal( - []artifact.ArtifactLayer{{ - FromImage: "luet/base", - ToImage: "test", - Diffs: artifact.ArtifactDiffs{ - Additions: artifacts, - }, - }})) - - opts2 = backend.Options{ - ImageName: "test", - SourcePath: tmpdir, - DockerFileName: "LuetDockerfile", - Destination: filepath.Join(tmpdir, "output3.tar"), - } - - Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred()) - Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue()) - Expect(b.ImageExists(opts2.ImageName)).To(BeFalse()) }) It("Detects available images", func() { diff --git a/pkg/compiler/backend/simpleimg.go b/pkg/compiler/backend/simpleimg.go index 2c875c1c..e181529a 100644 --- a/pkg/compiler/backend/simpleimg.go +++ b/pkg/compiler/backend/simpleimg.go @@ -20,6 +20,8 @@ import ( "os/exec" "strings" + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/mudler/luet/pkg/api/core/types" bus "github.com/mudler/luet/pkg/bus" @@ -70,6 +72,29 @@ func (s *SimpleImg) RemoveImage(opts Options) error { return nil } +func (s *SimpleImg) ImageReference(a string) (v1.Image, error) { + + f, err := s.ctx.Config.GetSystem().TempFile("snapshot") + if err != nil { + return nil, err + } + buildarg := []string{"save", a, "-o", f.Name()} + s.ctx.Spinner() + defer s.ctx.SpinnerStop() + + out, err := exec.Command("img", buildarg...).CombinedOutput() + if err != nil { + return nil, errors.Wrap(err, "Failed saving image: "+string(out)) + } + + img, err := crane.Load(f.Name()) + if err != nil { + return nil, err + } + + return img, nil +} + func (s *SimpleImg) DownloadImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePrePull, opts) diff --git a/pkg/compiler/backend_test.go b/pkg/compiler/backend_test.go deleted file mode 100644 index 6aa7768f..00000000 --- a/pkg/compiler/backend_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright © 2019 Ettore Di Giacinto -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, see . - -package compiler_test - -import ( - "github.com/mudler/luet/pkg/api/core/types" - . "github.com/mudler/luet/pkg/compiler" - . "github.com/mudler/luet/pkg/compiler/backend" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Docker image diffs", func() { - var b CompilerBackend - ctx := types.NewContext() - BeforeEach(func() { - b = NewSimpleDockerBackend(ctx) - }) - - Context("Generate diffs from docker images", func() { - It("Detect no changes", func() { - opts := Options{ - ImageName: "alpine:latest", - } - err := b.DownloadImage(opts) - Expect(err).ToNot(HaveOccurred()) - - layers, err := GenerateChanges(ctx, b, opts, opts) - Expect(err).ToNot(HaveOccurred()) - Expect(len(layers)).To(Equal(1)) - Expect(len(layers[0].Diffs.Additions)).To(Equal(0)) - Expect(len(layers[0].Diffs.Changes)).To(Equal(0)) - Expect(len(layers[0].Diffs.Deletions)).To(Equal(0)) - }) - - It("Detects additions and changed files", func() { - err := b.DownloadImage(Options{ - ImageName: "quay.io/mocaccino/micro", - }) - Expect(err).ToNot(HaveOccurred()) - err = b.DownloadImage(Options{ - ImageName: "quay.io/mocaccino/extra", - }) - Expect(err).ToNot(HaveOccurred()) - - layers, err := GenerateChanges(ctx, b, Options{ - ImageName: "quay.io/mocaccino/micro", - }, Options{ - ImageName: "quay.io/mocaccino/extra", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(layers)).To(Equal(1)) - - Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue()) - Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue()) - Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue()) - - Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue()) - Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue()) - Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue()) - - Expect(len(layers[0].Diffs.Deletions)).To(Equal(0)) - }) - }) -}) diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 4e346de0..d4df2c1a 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -29,6 +29,7 @@ import ( "sync" "time" + "github.com/mudler/luet/pkg/api/core/image" "github.com/mudler/luet/pkg/api/core/types" artifact "github.com/mudler/luet/pkg/api/core/types/artifact" bus "github.com/mudler/luet/pkg/bus" @@ -280,23 +281,36 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co } cs.Options.Context.Info(pkgTag, ":hammer: Generating delta") - diffs, err := GenerateChanges(cs.Options.Context, cs.Backend, builderOpts, runnerOpts) + + ref, err := cs.Backend.ImageReference(builderOpts.ImageName) if err != nil { - return nil, errors.Wrap(err, "Could not generate changes from layers") + return nil, err } - cs.Options.Context.Debug("Extracting image to grab files from delta") - if err := cs.Backend.ExtractRootfs(backend.Options{ - ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil { - return nil, errors.Wrap(err, "Could not extract rootfs") - } - artifact, err := artifact.ExtractArtifactFromDelta(cs.Options.Context, rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType) + ref2, err := cs.Backend.ImageReference(runnerOpts.ImageName) if err != nil { - return nil, errors.Wrap(err, "Could not generate deltas") + return nil, err } - artifact.CompileSpec = p - return artifact, nil + diff, err := image.Delta(ref, ref2) + if err != nil { + return nil, err + } + + // TODO: includes/excludes might need to get "/" stripped from prefix + a, err := artifact.ImageToArtifact( + cs.Options.Context, + ref2, + cs.Options.CompressionType, + p.Rel(fmt.Sprintf("%s%s", p.GetPackage().GetFingerPrint(), ".package.tar")), + image.ExtractDeltaFiles(cs.Options.Context, diff, p.GetIncludes(), p.GetExcludes()), + ) + if err != nil { + return nil, err + } + + a.CompileSpec = p + return a, nil } func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string, diff --git a/tests/integration/01_rootfs.sh b/tests/integration/01_rootfs.sh index 4973d0e1..d68ff990 100755 --- a/tests/integration/01_rootfs.sh +++ b/tests/integration/01_rootfs.sh @@ -12,7 +12,7 @@ oneTimeTearDown() { testBuild() { mkdir $tmpdir/testbuild - luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c > /dev/null + luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c buildst=$? assertEquals 'builds successfully' "$buildst" "0" assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"