Add crane-based methods for extraction

- create a new api package to encapsulate image manipulation
- use new api method to calculate delta

Fixes #258
Fixes #204
Fixes #90
This commit is contained in:
Ettore Di Giacinto
2021-10-23 18:44:48 +02:00
parent d44befe9ff
commit 6a9f19941a
13 changed files with 481 additions and 556 deletions

111
pkg/api/core/image/delta.go Normal file
View File

@@ -0,0 +1,111 @@
// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package image
import (
"archive/tar"
"io"
"regexp"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/mutate"
)
func compileRegexes(regexes []string) []*regexp.Regexp {
var result []*regexp.Regexp
for _, i := range regexes {
r, e := regexp.Compile(i)
if e != nil {
continue
}
result = append(result, r)
}
return result
}
type ImageDiffNode struct {
Name string `json:"Name"`
Size int `json:"Size"`
}
type ImageDiff struct {
Additions []ImageDiffNode `json:"Adds"`
Deletions []ImageDiffNode `json:"Dels"`
Changes []ImageDiffNode `json:"Mods"`
}
func Delta(srcimg, dstimg v1.Image) (res ImageDiff, err error) {
srcReader := mutate.Extract(srcimg)
defer srcReader.Close()
dstReader := mutate.Extract(dstimg)
defer dstReader.Close()
filesSrc, filesDst := map[string]int64{}, map[string]int64{}
srcTar := tar.NewReader(srcReader)
dstTar := tar.NewReader(dstReader)
for {
var hdr *tar.Header
hdr, err = srcTar.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return
}
filesSrc[hdr.Name] = hdr.Size
}
for {
var hdr *tar.Header
hdr, err = dstTar.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return
}
filesDst[hdr.Name] = hdr.Size
}
err = nil
for f, size := range filesDst {
if size2, exist := filesSrc[f]; exist && size2 != size {
res.Changes = append(res.Changes, ImageDiffNode{
Name: f,
Size: int(size),
})
} else if !exist {
res.Additions = append(res.Additions, ImageDiffNode{
Name: f,
Size: int(size),
})
}
}
for f, size := range filesSrc {
if _, exist := filesDst[f]; !exist {
res.Deletions = append(res.Deletions, ImageDiffNode{
Name: f,
Size: int(size),
})
}
}
return
}

View File

@@ -0,0 +1,136 @@
// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package image_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
daemon "github.com/google/go-containerregistry/pkg/v1/daemon"
. "github.com/mudler/luet/pkg/api/core/image"
"github.com/mudler/luet/pkg/api/core/types"
"github.com/mudler/luet/pkg/helpers/file"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("DeltaInfo", func() {
Context("Generates deltas of images", func() {
It("Correctly detect packages", func() {
ref, err := name.ParseReference("alpine")
Expect(err).ToNot(HaveOccurred())
img, err := daemon.Image(ref)
Expect(err).ToNot(HaveOccurred())
layers, err := Delta(img, img)
Expect(err).ToNot(HaveOccurred())
Expect(len(layers.Changes)).To(Equal(0))
Expect(len(layers.Additions)).To(Equal(0))
Expect(len(layers.Deletions)).To(Equal(0))
})
Context("ExtractDeltaFiles", func() {
ctx := types.NewContext()
var tmpfile *os.File
var ref, ref2 name.Reference
var img, img2 v1.Image
var diff ImageDiff
var err error
BeforeEach(func() {
ctx = types.NewContext()
tmpfile, err = ioutil.TempFile("", "delta")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpfile.Name()) // clean up
ref, err = name.ParseReference("alpine")
Expect(err).ToNot(HaveOccurred())
ref2, err = name.ParseReference("golang:alpine")
Expect(err).ToNot(HaveOccurred())
img, err = daemon.Image(ref)
Expect(err).ToNot(HaveOccurred())
img2, err = daemon.Image(ref2)
Expect(err).ToNot(HaveOccurred())
diff, err = Delta(img, img2)
Expect(err).ToNot(HaveOccurred())
Expect(len(diff.Additions) > 0).To(BeTrue())
Expect(len(diff.Changes) > 0).To(BeTrue())
Expect(len(diff.Deletions) > 0).To(BeTrue())
})
It("Extract all deltas", func() {
tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{}, []string{}),
)
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
fmt.Println(file.ListDir(tmpdir))
Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeTrue())
Expect(file.Exists(filepath.Join(tmpdir, "bin", "sh"))).To(BeFalse())
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeTrue())
})
It("Extract deltas and excludes stuff", func() {
tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}),
)
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
})
It("Extract deltas and excludes stuff", func() {
tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}),
)
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go", "bin"))).To(BeFalse())
})
It("Extract deltas and excludes stuff", func() {
tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}),
)
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeTrue())
Expect(file.Exists(filepath.Join(tmpdir, "root", ".cache"))).To(BeFalse())
})
})
})
})

View File

@@ -0,0 +1,119 @@
// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package image
import (
"archive/tar"
"context"
containerdarchive "github.com/containerd/containerd/archive"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/mudler/luet/pkg/api/core/types"
"github.com/pkg/errors"
)
// Extract dir:
// -> First extract delta considering the dir
// Afterward create artifact pointing to the dir
// ExtractDeltaFiles returns an handler to extract files in a list
func ExtractDeltaFiles(
ctx *types.Context,
d ImageDiff,
includes []string, excludes []string,
) func(h *tar.Header) (bool, error) {
includeRegexp := compileRegexes(includes)
excludeRegexp := compileRegexes(excludes)
return func(h *tar.Header) (bool, error) {
switch {
case len(includes) == 0 && len(excludes) != 0:
for _, a := range d.Additions {
if h.Name == a.Name {
for _, i := range excludeRegexp {
if i.MatchString(a.Name) && h.Name == a.Name {
return false, nil
}
}
ctx.Info("Adding name", h.Name)
return true, nil
}
}
return false, nil
case len(includes) > 0 && len(excludes) == 0:
for _, a := range d.Additions {
for _, i := range includeRegexp {
if i.MatchString(a.Name) && h.Name == a.Name {
ctx.Info("Adding name", h.Name)
return true, nil
}
}
}
return false, nil
case len(includes) != 0 && len(excludes) != 0:
for _, a := range d.Additions {
for _, i := range includeRegexp {
if i.MatchString(a.Name) && h.Name == a.Name {
for _, e := range excludeRegexp {
if e.MatchString(a.Name) {
return false, nil
}
}
ctx.Info("Adding name", h.Name)
return true, nil
}
}
}
return false, nil
default:
for _, a := range d.Additions {
if h.Name == a.Name {
ctx.Info("Adding name", h.Name)
return true, nil
}
}
return false, nil
}
}
}
func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
src := mutate.Extract(img)
defer src.Close()
tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
if err != nil {
return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
if filter != nil {
opts = append(opts, containerdarchive.WithFilter(filter))
}
_, err = containerdarchive.Apply(context.Background(), tmpdiffs, src, opts...)
if err != nil {
return "", err
}
return tmpdiffs, nil
}

View File

@@ -0,0 +1,28 @@
// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package image_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestMutator(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Mutator API Suite")
}

View File

@@ -27,16 +27,16 @@ import (
"os"
"path"
"path/filepath"
"regexp"
v1 "github.com/google/go-containerregistry/pkg/v1"
zstd "github.com/klauspost/compress/zstd"
gzip "github.com/klauspost/pgzip"
//"strconv"
"strings"
"sync"
config "github.com/mudler/luet/pkg/api/core/config"
"github.com/mudler/luet/pkg/api/core/image"
types "github.com/mudler/luet/pkg/api/core/types"
bus "github.com/mudler/luet/pkg/bus"
backend "github.com/mudler/luet/pkg/compiler/backend"
@@ -67,6 +67,22 @@ type PackageArtifact struct {
Runtime *pkg.DefaultPackage `json:"runtime,omitempty"`
}
func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
tmpdiffs, err := image.Extract(ctx, img, filter)
if err != nil {
return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(tmpdiffs) // clean up
a := NewPackageArtifact(output)
a.CompressionType = t
err = a.Compress(tmpdiffs, 1)
if err != nil {
return nil, errors.Wrap(err, "Error met while creating package archive")
}
return a, nil
}
func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
copy := *p
return &copy
@@ -626,181 +642,3 @@ func (a *PackageArtifact) FileList() ([]string, error) {
}
return files, nil
}
type CopyJob struct {
Src, Dst string
Artifact string
}
func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) {
defer wg.Done()
for job := range s {
_, err := os.Lstat(job.Dst)
if err != nil {
ctx.Debug("Copying ", job.Src)
if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
ctx.Warning("Error copying", job, err)
}
}
}
}
func compileRegexes(regexes []string) []*regexp.Regexp {
var result []*regexp.Regexp
for _, i := range regexes {
r, e := regexp.Compile(i)
if e != nil {
continue
}
result = append(result, r)
}
return result
}
type ArtifactNode struct {
Name string `json:"Name"`
Size int `json:"Size"`
}
type ArtifactDiffs struct {
Additions []ArtifactNode `json:"Adds"`
Deletions []ArtifactNode `json:"Dels"`
Changes []ArtifactNode `json:"Mods"`
}
type ArtifactLayer struct {
FromImage string `json:"Image1"`
ToImage string `json:"Image2"`
Diffs ArtifactDiffs `json:"Diff"`
}
// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
archive, err := ctx.Config.GetSystem().TempDir("archive")
if err != nil {
return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
}
defer os.RemoveAll(archive) // clean up
if strings.HasSuffix(src, ".tar") {
rootfs, err := ctx.Config.GetSystem().TempDir("rootfs")
if err != nil {
return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(rootfs) // clean up
err = helpers.Untar(src, rootfs, keepPerms)
if err != nil {
return nil, errors.Wrap(err, "Error met while unpacking rootfs")
}
src = rootfs
}
toCopy := make(chan CopyJob)
var wg = new(sync.WaitGroup)
for i := 0; i < concurrency; i++ {
wg.Add(1)
go worker(ctx, i, wg, toCopy)
}
// Handle includes in spec. If specified they filter what gets in the package
if len(includes) > 0 && len(excludes) == 0 {
includeRegexp := compileRegexes(includes)
for _, l := range layers {
// Consider d.Additions (and d.Changes? - warn at least) only
ADDS:
for _, a := range l.Diffs.Additions {
for _, i := range includeRegexp {
if i.MatchString(a.Name) {
toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
continue ADDS
}
}
}
for _, a := range l.Diffs.Changes {
ctx.Debug("File ", a.Name, " changed")
}
for _, a := range l.Diffs.Deletions {
ctx.Debug("File ", a.Name, " deleted")
}
}
} else if len(includes) == 0 && len(excludes) != 0 {
excludeRegexp := compileRegexes(excludes)
for _, l := range layers {
// Consider d.Additions (and d.Changes? - warn at least) only
ADD:
for _, a := range l.Diffs.Additions {
for _, i := range excludeRegexp {
if i.MatchString(a.Name) {
continue ADD
}
}
toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
}
for _, a := range l.Diffs.Changes {
ctx.Debug("File ", a.Name, " changed")
}
for _, a := range l.Diffs.Deletions {
ctx.Debug("File ", a.Name, " deleted")
}
}
} else if len(includes) != 0 && len(excludes) != 0 {
includeRegexp := compileRegexes(includes)
excludeRegexp := compileRegexes(excludes)
for _, l := range layers {
// Consider d.Additions (and d.Changes? - warn at least) only
EXCLUDES:
for _, a := range l.Diffs.Additions {
for _, i := range includeRegexp {
if i.MatchString(a.Name) {
for _, e := range excludeRegexp {
if e.MatchString(a.Name) {
continue EXCLUDES
}
}
toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
continue EXCLUDES
}
}
}
for _, a := range l.Diffs.Changes {
ctx.Debug("File ", a.Name, " changed")
}
for _, a := range l.Diffs.Deletions {
ctx.Debug("File ", a.Name, " deleted")
}
}
} else {
// Otherwise just grab all
for _, l := range layers {
// Consider d.Additions (and d.Changes? - warn at least) only
for _, a := range l.Diffs.Additions {
ctx.Debug("File ", a.Name, " added")
toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
}
for _, a := range l.Diffs.Changes {
ctx.Debug("File ", a.Name, " changed")
}
for _, a := range l.Diffs.Deletions {
ctx.Debug("File ", a.Name, " deleted")
}
}
}
close(toCopy)
wg.Wait()
a := NewPackageArtifact(dst)
a.CompressionType = t
err = a.Compress(archive, concurrency)
if err != nil {
return nil, errors.Wrap(err, "Error met while creating package archive")
}
return a, nil
}

View File

@@ -22,7 +22,6 @@ import (
"github.com/mudler/luet/pkg/api/core/types"
. "github.com/mudler/luet/pkg/api/core/types/artifact"
"github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler/backend"
backend "github.com/mudler/luet/pkg/compiler/backend"
compression "github.com/mudler/luet/pkg/compiler/types/compression"
@@ -30,7 +29,6 @@ import (
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
. "github.com/mudler/luet/pkg/compiler"
helpers "github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/tree"
@@ -117,53 +115,7 @@ RUN echo bar > /test2`))
}
Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2)
Expect(err).ToNot(HaveOccurred())
artifacts := []ArtifactNode{{
Name: "/luetbuild/LuetDockerfile",
Size: 175,
}}
if os.Getenv("DOCKER_BUILDKIT") == "1" {
artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
}
artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
Expect(diffs).To(Equal(
[]ArtifactLayer{{
FromImage: "luet/base",
ToImage: "test",
Diffs: ArtifactDiffs{
Additions: artifacts,
},
}}))
err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
Expect(err).ToNot(HaveOccurred())
a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
Expect(err).ToNot(HaveOccurred())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
err = helpers.Untar(a.Path, unpacked, false)
Expect(err).ToNot(HaveOccurred())
Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("foo\n"))
content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
Expect(err).ToNot(HaveOccurred())
Expect(content2).To(Equal("bar\n"))
err = a.Hash()
Expect(err).ToNot(HaveOccurred())
err = a.Verify()
Expect(err).ToNot(HaveOccurred())
Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
err = a.Verify()
Expect(err).To(HaveOccurred())
})
It("Generates packages images", func() {