Use API also when pulling from helpers used in client

This commit is contained in:
Ettore Di Giacinto
2021-10-24 00:57:50 +02:00
parent ad489c2157
commit ebbb3aad27
17 changed files with 113 additions and 233 deletions

View File

@@ -62,11 +62,6 @@ func NewUnpackCommand() *cobra.Command {
identity, _ := cmd.Flags().GetString("auth-identity-token")
registryToken, _ := cmd.Flags().GetString("auth-registry-token")
temp, err := util.DefaultContext.Config.GetSystem().TempDir("contentstore")
if err != nil {
util.DefaultContext.Fatal("Cannot create a tempdir", err.Error())
}
util.DefaultContext.Info("Downloading", image, "to", destination)
auth := &types.AuthConfig{
Username: user,
@@ -77,7 +72,7 @@ func NewUnpackCommand() *cobra.Command {
RegistryToken: registryToken,
}
info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
info, err := docker.DownloadAndExtractDockerImage(util.DefaultContext, image, destination, auth, verify)
if err != nil {
util.DefaultContext.Error(err.Error())
os.Exit(1)

View File

@@ -81,7 +81,7 @@ var _ = Describe("Delta", func() {
})
It("Extract all deltas", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{}, []string{}),
@@ -96,7 +96,7 @@ var _ = Describe("Delta", func() {
})
It("Extract deltas and excludes /usr/local/go", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{}, []string{"usr/local/go"}),
@@ -106,7 +106,7 @@ var _ = Describe("Delta", func() {
Expect(file.Exists(filepath.Join(tmpdir, "usr", "local", "go"))).To(BeFalse())
})
It("Extract deltas and excludes /usr/local/go/bin, but includes /usr/local/go", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{"usr/local/go/bin"}),
@@ -118,7 +118,7 @@ var _ = Describe("Delta", func() {
})
It("Extract deltas and includes /usr/local/go", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img2,
ExtractDeltaFiles(ctx, diff, []string{"usr/local/go"}, []string{}),

View File

@@ -166,7 +166,7 @@ func ExtractFiles(
}
}
func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
defer reader.Close()
perms := map[string][]int{}
@@ -177,17 +177,17 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filt
perms[h.Name] = []int{h.Gid, h.Uid}
xattrs[h.Name] = h.Xattrs
paxrecords[h.Name] = h.PAXRecords
return filter(h)
if filter != nil {
return filter(h)
}
return true, nil
}
if filter != nil {
opts = append(opts, containerdarchive.WithFilter(f))
}
opts = append(opts, containerdarchive.WithFilter(f))
_, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
c, err := containerdarchive.Apply(context.Background(), output, reader, opts...)
if err != nil {
return "", err
return 0, "", err
}
for f, p := range perms {
@@ -212,17 +212,17 @@ func ExtractReader(ctx *types.Context, reader io.ReadCloser, output string, filt
}
}
return output, nil
return c, output, nil
}
func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
func Extract(ctx *types.Context, img v1.Image, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
if err != nil {
return "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
return 0, "", errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
return ExtractReader(ctx, mutate.Extract(img), tmpdiffs, filter, opts...)
}
func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (string, error) {
func ExtractTo(ctx *types.Context, img v1.Image, output string, filter func(h *tar.Header) (bool, error), opts ...containerdarchive.ApplyOpt) (int64, string, error) {
return ExtractReader(ctx, mutate.Extract(img), output, filter, opts...)
}

View File

@@ -55,7 +55,7 @@ var _ = Describe("Extract", func() {
})
It("Extract all files", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img,
ExtractFiles(ctx, "", []string{}, []string{}),
@@ -68,7 +68,7 @@ var _ = Describe("Extract", func() {
})
It("Extract specific dir", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img,
ExtractFiles(ctx, "/usr", []string{}, []string{}),
@@ -81,7 +81,7 @@ var _ = Describe("Extract", func() {
})
It("Extract a dir with includes/excludes", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img,
ExtractFiles(ctx, "/usr", []string{"bin"}, []string{"sbin"}),
@@ -95,7 +95,7 @@ var _ = Describe("Extract", func() {
})
It("Extract with includes/excludes", func() {
tmpdir, err := Extract(
_, tmpdir, err := Extract(
ctx,
img,
ExtractFiles(ctx, "", []string{"/usr|/usr/bin"}, []string{"^/bin"}),

View File

@@ -68,7 +68,7 @@ type PackageArtifact struct {
}
func ImageToArtifact(ctx *types.Context, img v1.Image, t compression.Implementation, output string, filter func(h *tar.Header) (bool, error)) (*PackageArtifact, error) {
tmpdiffs, err := image.Extract(ctx, img, filter)
_, tmpdiffs, err := image.Extract(ctx, img, filter)
if err != nil {
return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}

View File

@@ -20,6 +20,7 @@ import (
"os"
"path/filepath"
"github.com/mudler/luet/pkg/api/core/image"
"github.com/mudler/luet/pkg/api/core/types"
. "github.com/mudler/luet/pkg/api/core/types/artifact"
. "github.com/mudler/luet/pkg/compiler/backend"
@@ -155,7 +156,19 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(result) // clean up
err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
img, err := b.ImageReference(resultingImage)
Expect(err).ToNot(HaveOccurred())
_, _, err = image.ExtractTo(
ctx,
img,
resultingImage,
image.ExtractFiles(
ctx,
"",
[]string{},
[]string{},
),
)
Expect(err).ToNot(HaveOccurred())
content, err := ioutil.ReadFile(filepath.Join(result, "test"))
@@ -197,7 +210,19 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(result) // clean up
err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
img, err := b.ImageReference(resultingImage)
Expect(err).ToNot(HaveOccurred())
_, _, err = image.ExtractTo(
ctx,
img,
resultingImage,
image.ExtractFiles(
ctx,
"",
[]string{},
[]string{},
),
)
Expect(err).ToNot(HaveOccurred())
Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())

View File

@@ -28,7 +28,6 @@ type CompilerBackend interface {
ExportImage(backend.Options) error
RemoveImage(backend.Options) error
ImageDefinitionToTar(backend.Options) error
ExtractRootfs(opts backend.Options, keepPerms bool) error
CopyImage(string, string) error
DownloadImage(opts backend.Options) error

View File

@@ -16,23 +16,14 @@
package backend
import (
"encoding/json"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/daemon"
"github.com/mudler/luet/pkg/api/core/types"
bus "github.com/mudler/luet/pkg/bus"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
v1 "github.com/google/go-containerregistry/pkg/v1"
capi "github.com/mudler/docker-companion/api"
"github.com/mudler/luet/pkg/helpers"
"github.com/pkg/errors"
)
@@ -195,102 +186,3 @@ func (s *SimpleDocker) ExportImage(opts Options) error {
type ManifestEntry struct {
Layers []string `json:"Layers"`
}
func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
name := opts.ImageName
dst := opts.Destination
if !b.ImageExists(name) {
if err := b.DownloadImage(opts); err != nil {
return errors.Wrap(err, "failed pulling image "+name+" during extraction")
}
}
tempexport, err := ioutil.TempDir(dst, "tmprootfs")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(tempexport) // clean up
imageExport := filepath.Join(tempexport, "image.tar")
b.ctx.Spinner()
defer b.ctx.SpinnerStop()
if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
return errors.Wrap(err, "failed while extracting rootfs for "+name)
}
src := imageExport
if src == "" && opts.ImageName != "" {
tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(tempUnpack) // clean up
imageExport := filepath.Join(tempUnpack, "image.tar")
if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
return errors.Wrap(err, "while exporting image before extraction")
}
src = imageExport
}
rootfs, err := ioutil.TempDir(dst, "tmprootfs")
if err != nil {
return errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(rootfs) // clean up
// TODO: Following as option if archive as output?
// archive, err := ioutil.TempDir(os.TempDir(), "archive")
// if err != nil {
// return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
// }
// defer os.RemoveAll(archive) // clean up
err = helpers.Untar(src, rootfs, keepPerms)
if err != nil {
return errors.Wrap(err, "Error met while unpacking rootfs")
}
manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
if err != nil {
return errors.Wrap(err, "Error met while reading image manifest")
}
// Unpack all layers
var manifestData []ManifestEntry
if err := json.Unmarshal([]byte(manifest), &manifestData); err != nil {
return errors.Wrap(err, "Error met while unmarshalling manifest")
}
layers_sha := []string{}
for _, data := range manifestData {
for _, l := range data.Layers {
if strings.Contains(l, "layer.tar") {
layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
}
}
}
// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
export, err := capi.CreateExport(rootfs)
if err != nil {
return err
}
err = export.UnPackLayers(layers_sha, dst, "containerd")
if err != nil {
return err
}
// err = helpers.Tar(archive, dst)
// if err != nil {
// return nil, errors.Wrap(err, "Error met while creating package archive")
// }
return nil
}

View File

@@ -16,7 +16,6 @@
package backend
import (
"os"
"os/exec"
"strings"
@@ -178,33 +177,6 @@ func (s *SimpleImg) ExportImage(opts Options) error {
return nil
}
// ExtractRootfs extracts the docker image content inside the destination
func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
name := opts.ImageName
path := opts.Destination
if !s.ImageExists(name) {
if err := s.DownloadImage(opts); err != nil {
return errors.Wrap(err, "failed pulling image "+name+" during extraction")
}
}
os.RemoveAll(path)
buildarg := []string{"unpack", "-o", path, name}
s.ctx.Debug(":tea: Extracting image " + name)
s.ctx.Spinner()
defer s.ctx.SpinnerStop()
out, err := exec.Command("img", buildarg...).CombinedOutput()
if err != nil {
return errors.Wrap(err, "Failed extracting image: "+string(out))
}
s.ctx.Debug(":tea: Image " + name + " extracted")
return nil
}
func (s *SimpleImg) Push(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePrePush, opts)

View File

@@ -232,8 +232,20 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
return nil, err
}
// artifact.ImageToArtifact(
// cs.Options.Context,
// img,
// cs.Options.CompressionType,
// p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"),
// image.ExtractFiles(
// cs.Options.Context,
// strings.TrimLeft(p.GetPackageDir(), "/"),
// p.GetIncludes(),
// p.GetExcludes(),
// ),
// )
// TODO: Trim includes/excludes from "/" ?
rootfs, err := image.Extract(
_, rootfs, err := image.Extract(
cs.Options.Context,
img,
image.ExtractFiles(

View File

@@ -20,12 +20,13 @@ import (
"encoding/hex"
"fmt"
"os"
"strings"
"github.com/containerd/containerd/images"
luetimages "github.com/mudler/luet/pkg/api/core/image"
luettypes "github.com/mudler/luet/pkg/api/core/types"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
continerdarchive "github.com/containerd/containerd/archive"
"github.com/docker/cli/cli/trust"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
@@ -34,7 +35,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/mudler/luet/pkg/bus"
"github.com/opencontainers/go-digest"
@@ -166,7 +166,7 @@ func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.
}
// DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
func DownloadAndExtractDockerImage(ctx *luettypes.Context, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
if verify {
img, err := verifyImage(image, auth)
if err != nil {
@@ -206,13 +206,20 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
return nil, err
}
reader := mutate.Extract(img)
defer reader.Close()
defer os.RemoveAll(temp)
bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
c, err := continerdarchive.Apply(context.TODO(), dest, reader)
var c int64
c, _, err = luetimages.ExtractTo(
ctx,
img,
dest,
luetimages.ExtractFiles(
ctx,
"",
[]string{},
[]string{},
),
)
if err != nil {
return nil, err
}
@@ -229,7 +236,3 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
},
}, nil
}
func StripInvalidStringsFromImage(s string) string {
return strings.ReplaceAll(s, "+", "-")
}

View File

@@ -1,30 +0,0 @@
// Copyright © 2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package docker_test
import (
"github.com/mudler/luet/pkg/helpers/docker"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("StripInvalidStringsFromImage", func() {
Context("Image names", func() {
It("strips invalid chars", func() {
Expect(docker.StripInvalidStringsFromImage("foo+bar")).To(Equal("foo-bar"))
})
})
})

View File

@@ -17,8 +17,9 @@
package helpers
import (
"github.com/asaskevich/govalidator"
"strings"
"github.com/asaskevich/govalidator"
)
func StripRegistryFromImage(image string) string {
@@ -28,3 +29,7 @@ func StripRegistryFromImage(image string) string {
}
return image
}
func SanitizeImageString(s string) string {
return strings.ReplaceAll(s, "+", "-")
}

View File

@@ -23,6 +23,11 @@ import (
)
var _ = Describe("Helpers", func() {
Context("Image names", func() {
It("strips invalid chars", func() {
Expect(SanitizeImageString("foo+bar")).To(Equal("foo-bar"))
})
})
Context("StripRegistryFromImage", func() {
It("Strips the domain name", func() {
out := StripRegistryFromImage("valid.domain.org/base/image:tag")

View File

@@ -28,6 +28,7 @@ import (
luetTypes "github.com/mudler/luet/pkg/api/core/types"
"github.com/mudler/luet/pkg/api/core/types/artifact"
"github.com/mudler/luet/pkg/helpers"
"github.com/mudler/luet/pkg/helpers/docker"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
@@ -104,14 +105,8 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID())
c.context.Info("Downloading image", imageName)
contentstore, err := c.context.Config.GetSystem().TempDir("contentstore")
if err != nil {
c.context.Warning("Cannot create contentstore", err.Error())
continue
}
// imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName())
info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
if err != nil {
c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
continue
@@ -157,7 +152,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
func (c *DockerClient) DownloadFile(name string) (string, error) {
var file *os.File = nil
var err error
var temp, contentstore string
var temp string
// Files should be in URI/repository:<file>
ok := false
@@ -172,16 +167,10 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
continue
}
contentstore, err = c.context.Config.GetSystem().TempDir("contentstore")
if err != nil {
c.context.Warning("Cannot create contentstore", err.Error())
continue
}
imageName := fmt.Sprintf("%s:%s", uri, docker.StripInvalidStringsFromImage(name))
imageName := fmt.Sprintf("%s:%s", uri, helpers.SanitizeImageString(name))
c.context.Info("Downloading", imageName)
info, err := docker.DownloadAndExtractDockerImage(contentstore, imageName, temp, c.auth, c.RepoData.Verify)
info, err := docker.DownloadAndExtractDockerImage(c.context, imageName, temp, c.auth, c.RepoData.Verify)
if err != nil {
c.context.Warning(fmt.Sprintf(errImageDownloadMsg, imageName, err.Error()))
continue

View File

@@ -24,13 +24,13 @@ import (
"strings"
"time"
"github.com/mudler/luet/pkg/api/core/image"
"github.com/mudler/luet/pkg/api/core/types"
artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
"github.com/mudler/luet/pkg/bus"
compiler "github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/helpers"
"github.com/mudler/luet/pkg/helpers/docker"
pkg "github.com/mudler/luet/pkg/package"
"github.com/pkg/errors"
@@ -165,7 +165,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt
if err != nil {
return errors.Wrap(err, "failed generating checksums for tree")
}
imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, docker.StripInvalidStringsFromImage(a.GetFileName()))
imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, helpers.SanitizeImageString(a.GetFileName()))
if checkIfExists && d.imagePush && d.b.ImageAvailable(imageTree) && !d.force {
d.context.Info("Image", imageTree, "already present, skipping. use --force-push to override")
return nil
@@ -191,13 +191,26 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
defer os.RemoveAll(repoTemp) // clean up
if r.GetBackend().ImageAvailable(imageRepository) {
if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil {
img, err := r.GetBackend().ImageReference(imageRepository)
if err != nil {
return errors.Wrapf(err, "while downloading '%s'", imageRepository)
}
if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
_, _, err = image.ExtractTo(
d.context,
img,
repoTemp,
image.ExtractFiles(
d.context,
"",
[]string{},
[]string{},
),
)
if err != nil {
return errors.Wrapf(err, "while extracting '%s'", imageRepository)
}
}
repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)

View File

@@ -27,9 +27,9 @@ import (
"strconv"
"strings"
"github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"github.com/mudler/luet/pkg/helpers/docker"
"github.com/mudler/luet/pkg/helpers/match"
version "github.com/mudler/luet/pkg/versioner"
@@ -350,7 +350,7 @@ func (p *DefaultPackage) GetPackageName() string {
}
func (p *DefaultPackage) ImageID() string {
return docker.StripInvalidStringsFromImage(p.GetFingerPrint())
return helpers.SanitizeImageString(p.GetFingerPrint())
}
// GetBuildTimestamp returns the package build timestamp