// Copyright © 2020 Ettore Di Giacinto // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License along // with this program; if not, see . package client import ( "fmt" "os" "path" "path/filepath" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" "github.com/docker/docker/pkg/archive" "github.com/docker/go-units" "github.com/moby/buildkit/session" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/genuinetools/img/client" "github.com/genuinetools/img/types" "github.com/moby/buildkit/util/appcontext" "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" ) type DockerClient struct { RepoData RepoData } func NewDockerClient(r RepoData) *DockerClient { return &DockerClient{RepoData: r} } func downloadAndExtractDockerImage(image, dest string) error { temp, err := config.LuetCfg.GetSystem().TempDir("contentstore") if err != nil { return err } defer os.RemoveAll(temp) Debug("Temporary directory", temp) c, err := client.New(temp, types.NativeBackend, nil) if err != nil { return errors.Wrapf(err, "failed creating client") } defer c.Close() // Slightly adapted from https://github.com/genuinetools/img/blob/54d0ca981c1260546d43961a538550eef55c87cf/pull.go var listedImage *client.ListedImage // Create the context. ctx := appcontext.Context() sess, sessDialer, err := c.Session(ctx) if err != nil { return errors.Wrapf(err, "failed creating Session") } ctx = session.NewContext(ctx, sess.ID()) ctx = namespaces.WithNamespace(ctx, "buildkit") Debug("Starting session") go func() { sess.Run(ctx, sessDialer) }() defer func() { Debug("Closing session") sess.Close() Debug("Session closed") }() Debug("Pulling image", image) listedImage, err = c.Pull(ctx, image) if err != nil { return errors.Wrapf(err, "failed listing images") } Debug("Pulled:", listedImage.Target.Digest) Debug("Size:", units.BytesSize(float64(listedImage.ContentSize))) Debug("Unpacking", image, "to", dest) os.RemoveAll(dest) // XXX: Unpacking stalls. See why calling img works, and with luet doesn't. Shall we unpack by reimplementing the client here? // err = c.Unpack(ctx, image, dest) // Debug("Finished Unpacking") // opt, err := c.createWorkerOpt(false) // if err != nil { // return fmt.Errorf("creating worker opt failed: %v", err) // } img, err := opt.ImageStore.Get(ctx, image) if err != nil { return fmt.Errorf("getting image %s from image store failed: %v", image, err) } manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default()) if err != nil { return fmt.Errorf("getting image manifest failed: %v", err) } for _, desc := range manifest.Layers { logrus.Debugf("Unpacking layer %s", desc.Digest.String()) // Read the blob from the content store. layer, err := opt.ContentStore.ReaderAt(ctx, desc) if err != nil { return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err) } // Unpack the tarfile to the rootfs path. // FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{ NoLchown: true, }); err != nil { return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err) } } return errors.Wrapf(err, "failed unpacking images") // eg, ctx := errgroup.WithContext(ctx) // eg.Go(func() error { // return sess.Run(ctx, sessDialer) // }) // eg.Go(func() error { // defer sess.Close() // var err error // listedImage, err = c.Pull(ctx, image) // if err != nil { // return errors.Wrapf(err, "failed listing images") // } // os.RemoveAll(dest) // return errors.Wrapf(c.Unpack(ctx, image, dest), "failed unpacking images") // }) // if err := eg.Wait(); err != nil { // return err // } //Debug("Pulled:", listedImage.Target.Digest) // Debug("Size:", units.BytesSize(float64(listedImage.ContentSize))) // Get the identifier for the image. // id, err := source.NewImageIdentifier(image) // if err != nil { // return err // } // Debug("Image identifier", id.ID()) // named, err := reference.ParseNormalizedNamed(image) // if err != nil { // return fmt.Errorf("parsing image name %q failed: %v", image, err) // } // // Add the latest lag if they did not provide one. // named = reference.TagNameOnly(named) // image = named.String() // ctx := appcontext.Context() // sess, sessDialer, err := c.Session(ctx) // if err != nil { // return err // } // ctx = session.NewContext(ctx, sess.ID()) // ctx = namespaces.WithNamespace(ctx, "buildkit") // snapshotRoot := filepath.Join(temp, "snapshots") // XXX: We force native backend. Our docker images will have just one layer as they are created from scratch. // No need to depend on FUSE/overlayfs available in the system // s, err := native.NewSnapshotter(snapshotRoot) // contentStore, err := local.NewStore(filepath.Join(temp, "content")) // if err != nil { // return err // } // // Open the bolt database for metadata. // db, err := bolt.Open(filepath.Join(temp, "containerdmeta.db"), 0644, nil) // if err != nil { // return err // } // // Create the new database for metadata. // mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{ // types.NativeBackend: s, // }) // if err := mdb.Init(ctx); err != nil { // return err // } // // Create the image store. // imageStore := ctdmetadata.NewImageStore(mdb) // contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit") // Debug("Getting image", image) // img, err := imageStore.Get(ctx, image) // if err != nil { // return fmt.Errorf("getting image %s from image store failed: %v", image, err) // } // manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default()) // if err != nil { // return fmt.Errorf("getting image manifest failed: %v", err) // } // for _, desc := range manifest.Layers { // Debug("Unpacking layer %s", desc.Digest.String()) // // Read the blob from the content store. // layer, err := contentStore.ReaderAt(ctx, desc) // if err != nil { // return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err) // } // // Unpack the tarfile to the rootfs path. // // FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions // if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{ // NoLchown: true, // ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted' // }); err != nil { // return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err) // } // } return nil } func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { //var u *url.URL = nil var err error var temp string var resultingArtifact compiler.Artifact artifactName := path.Base(artifact.GetPath()) cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) ok := false // TODO: // Files are in URI/packagename:version (GetPackageImageName() method) // use downloadAndExtract .. and egenrate an archive to consume. Checksum should be already checked while downloading the image // with the above functions, because Docker images already contain such metadata // - Check how verification is done when calling DownloadArtifact outside, similarly we need to check DownloadFile, and how verification // is done in such cases (see repository.go) // Check if file is already in cache if helpers.Exists(cacheFile) { Info("Use artifact", artifactName, "from cache.") } else { temp, err = config.LuetCfg.GetSystem().TempDir("tree") if err != nil { return nil, err } defer os.RemoveAll(temp) for _, uri := range c.RepoData.Urls { Debug("Downloading artifact", artifactName, "from", uri) imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().GetFingerPrint()) // imageName := fmt.Sprintf("%s/%s", uri, artifact.GetCompileSpec().GetPackage().GetPackageImageName()) err = downloadAndExtractDockerImage(imageName, temp) if err != nil { Debug("Failed download of image", imageName) continue } Debug("\nCompressing result ", filepath.Join(temp, artifactName), "to", cacheFile) // We discard checksum, that are checked while during pull and unpack newart := artifact newart.SetPath(cacheFile) err = newart.Compress(temp, 1) if err != nil { Debug("Failed compressing package", imageName) continue } resultingArtifact = newart ok = true break } if !ok { return nil, err } } return resultingArtifact, nil } func (c *DockerClient) DownloadFile(name string) (string, error) { var file *os.File = nil var err error var temp string // Files should be in URI/repository: ok := false temp, err = config.LuetCfg.GetSystem().TempDir("tree") if err != nil { return "", err } for _, uri := range c.RepoData.Urls { file, err = config.LuetCfg.GetSystem().TempFile("DockerClient") if err != nil { continue } Debug("Downloading file", name, "from", uri) imageName := fmt.Sprintf("%s:%s", uri, name) //imageName := fmt.Sprintf("%s/%s:%s", uri, "repository", name) err = downloadAndExtractDockerImage(imageName, temp) if err != nil { Debug("Failed download of image", imageName) continue } Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name()) err = helpers.CopyFile(filepath.Join(temp, name), file.Name()) if err != nil { continue } ok = true break } if !ok { return "", err } return file.Name(), err }