diff --git a/pkg/compiler/backend/simpledocker.go b/pkg/compiler/backend/simpledocker.go index 0767ad69..d3bc3139 100644 --- a/pkg/compiler/backend/simpledocker.go +++ b/pkg/compiler/backend/simpledocker.go @@ -240,7 +240,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP } } } - + // TODO: Drop capi in favor of the img approach already used in pkg/installer/repository export, err := capi.CreateExport(rootfs) if err != nil { return err diff --git a/pkg/installer/client/docker.go b/pkg/installer/client/docker.go new file mode 100644 index 00000000..2b51b2e6 --- /dev/null +++ b/pkg/installer/client/docker.go @@ -0,0 +1,356 @@ +// Copyright © 2020 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package client + +import ( + "fmt" + "os" + "path" + "path/filepath" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/platforms" + "github.com/docker/docker/pkg/archive" + "github.com/docker/go-units" + "github.com/moby/buildkit/session" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/genuinetools/img/client" + + "github.com/genuinetools/img/types" + "github.com/moby/buildkit/util/appcontext" + + "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/config" + "github.com/mudler/luet/pkg/helpers" + . "github.com/mudler/luet/pkg/logger" +) + +type DockerClient struct { + RepoData RepoData +} + +func NewDockerClient(r RepoData) *DockerClient { + return &DockerClient{RepoData: r} +} + +func downloadAndExtractDockerImage(image, dest string) error { + temp, err := config.LuetCfg.GetSystem().TempDir("contentstore") + if err != nil { + return err + } + defer os.RemoveAll(temp) + Debug("Temporary directory", temp) + c, err := client.New(temp, types.NativeBackend, nil) + if err != nil { + return errors.Wrapf(err, "failed creating client") + } + defer c.Close() + + // Slightly adapted from https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go + var listedImage *client.ListedImage + // Create the context. + ctx := appcontext.Context() + sess, sessDialer, err := c.Session(ctx) + if err != nil { + return errors.Wrapf(err, "failed creating Session") + } + ctx = session.NewContext(ctx, sess.ID()) + ctx = namespaces.WithNamespace(ctx, "buildkit") + + Debug("Starting session") + go func() { + sess.Run(ctx, sessDialer) + + }() + defer func() { + Debug("Closing session") + sess.Close() + Debug("Session closed") + }() + + Debug("Pulling image", image) + listedImage, err = c.Pull(ctx, image) + if err != nil { + return errors.Wrapf(err, "failed listing images") + + } + + Debug("Pulled:", listedImage.Target.Digest) + Debug("Size:", units.BytesSize(float64(listedImage.ContentSize))) + Debug("Unpacking", image, "to", dest) + os.RemoveAll(dest) + + // XXX: Unpacking stalls. See why calling img works, and with luet doesn't. Shall we unpack by reimplementing the client here? + + // err = c.Unpack(ctx, image, dest) + // Debug("Finished Unpacking") + + // opt, err := c.createWorkerOpt(false) + // if err != nil { + // return fmt.Errorf("creating worker opt failed: %v", err) + // } + + img, err := opt.ImageStore.Get(ctx, image) + if err != nil { + return fmt.Errorf("getting image %s from image store failed: %v", image, err) + } + + manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default()) + if err != nil { + return fmt.Errorf("getting image manifest failed: %v", err) + } + + for _, desc := range manifest.Layers { + logrus.Debugf("Unpacking layer %s", desc.Digest.String()) + + // Read the blob from the content store. + layer, err := opt.ContentStore.ReaderAt(ctx, desc) + if err != nil { + return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err) + } + + // Unpack the tarfile to the rootfs path. + // FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions + if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{ + NoLchown: true, + }); err != nil { + return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err) + } + } + + return errors.Wrapf(err, "failed unpacking images") + + // eg, ctx := errgroup.WithContext(ctx) + + // eg.Go(func() error { + // return sess.Run(ctx, sessDialer) + // }) + // eg.Go(func() error { + // defer sess.Close() + // var err error + // listedImage, err = c.Pull(ctx, image) + // if err != nil { + // return errors.Wrapf(err, "failed listing images") + + // } + // os.RemoveAll(dest) + // return errors.Wrapf(c.Unpack(ctx, image, dest), "failed unpacking images") + // }) + + // if err := eg.Wait(); err != nil { + // return err + // } + + //Debug("Pulled:", listedImage.Target.Digest) + // Debug("Size:", units.BytesSize(float64(listedImage.ContentSize))) + + // Get the identifier for the image. + // id, err := source.NewImageIdentifier(image) + // if err != nil { + // return err + // } + // Debug("Image identifier", id.ID()) + + // named, err := reference.ParseNormalizedNamed(image) + // if err != nil { + // return fmt.Errorf("parsing image name %q failed: %v", image, err) + // } + // // Add the latest lag if they did not provide one. + // named = reference.TagNameOnly(named) + // image = named.String() + + // ctx := appcontext.Context() + // sess, sessDialer, err := c.Session(ctx) + // if err != nil { + // return err + // } + // ctx = session.NewContext(ctx, sess.ID()) + // ctx = namespaces.WithNamespace(ctx, "buildkit") + + // snapshotRoot := filepath.Join(temp, "snapshots") + + // XXX: We force native backend. Our docker images will have just one layer as they are created from scratch. + // No need to depend on FUSE/overlayfs available in the system + // s, err := native.NewSnapshotter(snapshotRoot) + + // contentStore, err := local.NewStore(filepath.Join(temp, "content")) + // if err != nil { + // return err + // } + + // // Open the bolt database for metadata. + // db, err := bolt.Open(filepath.Join(temp, "containerdmeta.db"), 0644, nil) + // if err != nil { + // return err + // } + + // // Create the new database for metadata. + // mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{ + // types.NativeBackend: s, + // }) + // if err := mdb.Init(ctx); err != nil { + // return err + // } + + // // Create the image store. + // imageStore := ctdmetadata.NewImageStore(mdb) + + // contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit") + + // Debug("Getting image", image) + // img, err := imageStore.Get(ctx, image) + // if err != nil { + // return fmt.Errorf("getting image %s from image store failed: %v", image, err) + // } + + // manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default()) + // if err != nil { + // return fmt.Errorf("getting image manifest failed: %v", err) + // } + + // for _, desc := range manifest.Layers { + // Debug("Unpacking layer %s", desc.Digest.String()) + + // // Read the blob from the content store. + // layer, err := contentStore.ReaderAt(ctx, desc) + // if err != nil { + // return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err) + // } + + // // Unpack the tarfile to the rootfs path. + // // FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions + // if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{ + // NoLchown: true, + // ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted' + // }); err != nil { + // return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err) + // } + // } + return nil +} + +func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { + //var u *url.URL = nil + var err error + var temp string + var resultingArtifact compiler.Artifact + artifactName := path.Base(artifact.GetPath()) + cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) + ok := false + + // TODO: + // Files are in URI/packagename:version (GetPackageImageName() method) + // use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image + // with the above functions, because Docker images already contain such metadata + // - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification + // is done in such cases (see repository.go) + + // Check if file is already in cache + if helpers.Exists(cacheFile) { + Info("Use artifact", artifactName, "from cache.") + } else { + + temp, err = config.LuetCfg.GetSystem().TempDir("tree") + if err != nil { + return nil, err + } + defer os.RemoveAll(temp) + + for _, uri := range c.RepoData.Urls { + Debug("Downloading artifact", artifactName, "from", uri) + + imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint()) + + // imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName()) + err = downloadAndExtractDockerImage(imageName, temp) + if err != nil { + Debug("Failed download of image", imageName) + continue + } + Debug("\nCompressing result ", filepath.Join(temp, artifactName), "to", cacheFile) + + // We discard checksum, that are checked while during pull and unpack + newart := artifact + newart.SetPath(cacheFile) + err = newart.Compress(temp, 1) + if err != nil { + Debug("Failed compressing package", imageName) + continue + } + resultingArtifact = newart + + ok = true + break + } + + if !ok { + return nil, err + } + } + + return resultingArtifact, nil +} + +func (c *DockerClient) DownloadFile(name string) (string, error) { + var file *os.File = nil + var err error + var temp string + + // Files should be in URI/repository: + ok := false + + temp, err = config.LuetCfg.GetSystem().TempDir("tree") + if err != nil { + return "", err + } + + for _, uri := range c.RepoData.Urls { + + file, err = config.LuetCfg.GetSystem().TempFile("DockerClient") + if err != nil { + continue + } + + Debug("Downloading file", name, "from", uri) + + imageName := fmt.Sprintf("%s:%s", uri, name) + //imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name) + err = downloadAndExtractDockerImage(imageName, temp) + if err != nil { + Debug("Failed download of image", imageName) + continue + } + + Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name()) + err = helpers.CopyFile(filepath.Join(temp, name), file.Name()) + + if err != nil { + continue + } + ok = true + break + } + + if !ok { + return "", err + } + + return file.Name(), err +}