diff --git a/cmd/util.go b/cmd/util.go index 3320f615..f3790672 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -84,7 +84,7 @@ func NewUnpackCommand() *cobra.Command { os.Exit(1) } Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name)) - Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize)))) + Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size)))) }, } diff --git a/pkg/helpers/docker/docker.go b/pkg/helpers/docker/docker.go index 90222a7e..77eae5f3 100644 --- a/pkg/helpers/docker/docker.go +++ b/pkg/helpers/docker/docker.go @@ -24,7 +24,6 @@ import ( "github.com/containerd/containerd/images" fileHelper "github.com/mudler/luet/pkg/helpers/file" - "github.com/mudler/luet/pkg/helpers/imgworker" continerdarchive "github.com/containerd/containerd/archive" "github.com/docker/cli/cli/trust" @@ -130,31 +129,6 @@ type UnpackEventData struct { Dest string } -// privilegedExtractImage uses the imgworker (which requires privileges) to extract a container image -func privilegedExtractImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) { - defer os.RemoveAll(temp) - c, err := imgworker.New(temp, auth) - if err != nil { - return nil, errors.Wrapf(err, "failed creating client") - } - defer c.Close() - - listedImage, err := c.Pull(image) - if err != nil { - return nil, errors.Wrapf(err, "failed listing images") - } - - os.RemoveAll(dest) - - bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest}) - - err = c.Unpack(image, dest) - - bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest}) - - return listedImage, err -} - // UnarchiveLayers extract layers with archive.Untar from docker instead of containerd func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) { layers, err := img.Layers() @@ -192,7 +166,7 @@ func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types. } // DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode -func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) { +func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) { if verify { img, err := verifyImage(image, auth) if err != nil { @@ -201,10 +175,6 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon image = img } - if os.Getenv("LUET_PRIVILEGED_EXTRACT") == "true" { - return privilegedExtractImage(temp, image, dest, auth, verify) - } - if !fileHelper.Exists(dest) { if err := os.MkdirAll(dest, os.ModePerm); err != nil { return nil, errors.Wrapf(err, "cannot create destination directory") @@ -249,17 +219,14 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest}) - return &imgworker.ListedImage{ - Image: images.Image{ - Name: image, - Labels: m.Annotations, - Target: specs.Descriptor{ - MediaType: string(mt), - Digest: digest.Digest(d.String()), - Size: c, - }, + return &images.Image{ + Name: image, + Labels: m.Annotations, + Target: specs.Descriptor{ + MediaType: string(mt), + Digest: digest.Digest(d.String()), + Size: c, }, - ContentSize: c, }, nil } diff --git a/pkg/helpers/imgworker/auth.go b/pkg/helpers/imgworker/auth.go deleted file mode 100644 index c36a9fa9..00000000 --- a/pkg/helpers/imgworker/auth.go +++ /dev/null @@ -1,36 +0,0 @@ -package imgworker - -import ( - "context" - - "github.com/docker/docker/api/types" - - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth" - "google.golang.org/grpc" -) - -func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable { - return &authProvider{ - config: auth, - } -} - -type authProvider struct { - config *types.AuthConfig -} - -func (ap *authProvider) Register(server *grpc.Server) { - // no-op -} - -func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) { - res := &auth.CredentialsResponse{} - if ap.config.IdentityToken != "" { - res.Secret = ap.config.IdentityToken - } else { - res.Username = ap.config.Username - res.Secret = ap.config.Password - } - return res, nil -} diff --git a/pkg/helpers/imgworker/client.go b/pkg/helpers/imgworker/client.go deleted file mode 100644 index ceca0367..00000000 --- a/pkg/helpers/imgworker/client.go +++ /dev/null @@ -1,81 +0,0 @@ -package imgworker - -// FROM Slightly adapted from genuinetools/img worker - -import ( - "context" - "os" - "path/filepath" - - "github.com/containerd/containerd/namespaces" - dockertypes "github.com/docker/docker/api/types" - "github.com/genuinetools/img/types" - "github.com/moby/buildkit/control" - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/util/appcontext" - "github.com/moby/buildkit/worker/base" - "github.com/pkg/errors" -) - -// Client holds the information for the client we will use for communicating -// with the buildkit controller. -type Client struct { - backend string - localDirs map[string]string - root string - - sessionManager *session.Manager - controller *control.Controller - opts *base.WorkerOpt - - sess *session.Session - ctx context.Context - auth *dockertypes.AuthConfig -} - -// New returns a new client for communicating with the buildkit controller. -func New(root string, auth *dockertypes.AuthConfig) (*Client, error) { - // Native backend is fine, our images have just one layer. No need to depend on anything - backend := types.NativeBackend - - // Create the root/ - root = filepath.Join(root, "runc", backend) - if err := os.MkdirAll(root, 0700); err != nil { - return nil, err - } - c := &Client{ - backend: types.NativeBackend, - root: root, - localDirs: nil, - auth: auth, - } - - if err := c.prepare(); err != nil { - return nil, errors.Wrapf(err, "failed preparing client") - } - - // Create the start of the client. - return c, nil -} - -func (c *Client) Close() { - c.sess.Close() -} - -func (c *Client) prepare() error { - ctx := appcontext.Context() - sess, sessDialer, err := c.Session(ctx) - if err != nil { - return errors.Wrapf(err, "failed creating Session") - } - ctx = session.NewContext(ctx, sess.ID()) - ctx = namespaces.WithNamespace(ctx, "buildkit") - - c.ctx = ctx - c.sess = sess - - go func() { - sess.Run(ctx, sessDialer) - }() - return nil -} diff --git a/pkg/helpers/imgworker/pull.go b/pkg/helpers/imgworker/pull.go deleted file mode 100644 index 5a8b1935..00000000 --- a/pkg/helpers/imgworker/pull.go +++ /dev/null @@ -1,129 +0,0 @@ -package imgworker - -// FROM Slightly adapted from genuinetools/img worker - -import ( - "fmt" - - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" - "github.com/docker/distribution/reference" - "github.com/moby/buildkit/cache" - "github.com/moby/buildkit/exporter" - imageexporter "github.com/moby/buildkit/exporter/containerimage" - "github.com/moby/buildkit/source" - "github.com/moby/buildkit/source/containerimage" -) - -// ListedImage represents an image structure returuned from ListImages. -// It extends containerd/images.Image with extra fields. -type ListedImage struct { - images.Image - ContentSize int64 -} - -// Pull retrieves an image from a remote registry. -func (c *Client) Pull(image string) (*ListedImage, error) { - - ctx := c.ctx - - sm, err := c.getSessionManager() - if err != nil { - return nil, err - } - - // Parse the image name and tag. - named, err := reference.ParseNormalizedNamed(image) - if err != nil { - return nil, fmt.Errorf("parsing image name %q failed: %v", image, err) - } - // Add the latest lag if they did not provide one. - named = reference.TagNameOnly(named) - image = named.String() - - // Get the identifier for the image. - identifier, err := source.NewImageIdentifier(image) - if err != nil { - return nil, err - } - - // Create the worker opts. - opt, err := c.createWorkerOpt() - if err != nil { - return nil, fmt.Errorf("creating worker opt failed: %v", err) - } - - cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: opt.Snapshotter, - MetadataStore: opt.MetadataStore, - ContentStore: opt.ContentStore, - LeaseManager: opt.LeaseManager, - GarbageCollect: opt.GarbageCollect, - Applier: opt.Applier, - }) - if err != nil { - return nil, err - } - - // Create the source for the pull. - srcOpt := containerimage.SourceOpt{ - Snapshotter: opt.Snapshotter, - ContentStore: opt.ContentStore, - Applier: opt.Applier, - CacheAccessor: cm, - ImageStore: opt.ImageStore, - RegistryHosts: opt.RegistryHosts, - LeaseManager: opt.LeaseManager, - } - src, err := containerimage.NewSource(srcOpt) - if err != nil { - return nil, err - } - s, err := src.Resolve(ctx, identifier, sm) - if err != nil { - return nil, err - } - ref, err := s.Snapshot(ctx) - if err != nil { - return nil, err - } - - // Create the exporter for the pull. - iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{ - Snapshotter: opt.Snapshotter, - ContentStore: opt.ContentStore, - Differ: opt.Differ, - }) - if err != nil { - return nil, err - } - expOpt := imageexporter.Opt{ - SessionManager: sm, - ImageWriter: iw, - Images: opt.ImageStore, - RegistryHosts: opt.RegistryHosts, - LeaseManager: opt.LeaseManager, - } - exp, err := imageexporter.New(expOpt) - if err != nil { - return nil, err - } - e, err := exp.Resolve(ctx, map[string]string{"name": image}) - if err != nil { - return nil, err - } - if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil { - return nil, err - } - // Get the image. - img, err := opt.ImageStore.Get(ctx, image) - if err != nil { - return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err) - } - size, err := img.Size(ctx, opt.ContentStore, platforms.Default()) - if err != nil { - return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err) - } - - return &ListedImage{Image: img, ContentSize: size}, nil -} diff --git a/pkg/helpers/imgworker/session.go b/pkg/helpers/imgworker/session.go deleted file mode 100644 index 874feeac..00000000 --- a/pkg/helpers/imgworker/session.go +++ /dev/null @@ -1,49 +0,0 @@ -package imgworker - -// FROM Slightly adapted from genuinetools/img worker - -import ( - "context" - - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/filesync" - "github.com/moby/buildkit/session/testutil" - "github.com/pkg/errors" -) - -func (c *Client) getSessionManager() (*session.Manager, error) { - if c.sessionManager == nil { - var err error - c.sessionManager, err = session.NewManager() - if err != nil { - return nil, err - } - } - return c.sessionManager, nil -} - -// Session creates the session manager and returns the session and it's -// dialer. -func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) { - m, err := c.getSessionManager() - if err != nil { - return nil, nil, errors.Wrap(err, "failed to create session manager") - } - sessionName := "luet" - s, err := session.NewSession(ctx, sessionName, "") - if err != nil { - return nil, nil, errors.Wrap(err, "failed to create session") - } - syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs)) - for name, d := range c.localDirs { - syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d}) - } - s.Allow(filesync.NewFSSyncProvider(syncedDirs)) - s.Allow(NewDockerAuthProvider(c.auth)) - return s, sessionDialer(s, m), err -} - -func sessionDialer(s *session.Session, m *session.Manager) session.Dialer { - // FIXME: rename testutil - return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn))) -} diff --git a/pkg/helpers/imgworker/unpack.go b/pkg/helpers/imgworker/unpack.go deleted file mode 100644 index bd4cbf86..00000000 --- a/pkg/helpers/imgworker/unpack.go +++ /dev/null @@ -1,93 +0,0 @@ -package imgworker - -// FROM Slightly adapted from genuinetools/img worker - -import ( - "errors" - "fmt" - "github.com/mudler/luet/pkg/bus" - "os" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" - "github.com/docker/distribution/reference" - "github.com/docker/docker/pkg/archive" - "github.com/sirupsen/logrus" -) - -// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required. -// See how backends are unpacking images without asking for root permissions. - -// UnpackEventData is the data structure to pass for the bus events -type UnpackEventData struct { - Image string - Dest string -} - -// Unpack exports an image to a rootfs destination directory. -func (c *Client) Unpack(image, dest string) error { - - ctx := c.ctx - if len(dest) < 1 { - return errors.New("destination directory for rootfs cannot be empty") - } - - if _, err := os.Stat(dest); err == nil { - return fmt.Errorf("destination directory already exists: %s", dest) - } - - // Parse the image name and tag. - named, err := reference.ParseNormalizedNamed(image) - if err != nil { - return fmt.Errorf("parsing image name %q failed: %v", image, err) - } - // Add the latest lag if they did not provide one. - named = reference.TagNameOnly(named) - image = named.String() - - // Create the worker opts. - opt, err := c.createWorkerOpt() - if err != nil { - return fmt.Errorf("creating worker opt failed: %v", err) - } - - if opt.ImageStore == nil { - return errors.New("image store is nil") - } - - img, err := opt.ImageStore.Get(ctx, image) - if err != nil { - return fmt.Errorf("getting image %s from image store failed: %v", image, err) - } - - manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default()) - if err != nil { - return fmt.Errorf("getting image manifest failed: %v", err) - } - - _,_ = bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest}) - - for _, desc := range manifest.Layers { - logrus.Debugf("Unpacking layer %s", desc.Digest.String()) - - // Read the blob from the content store. - layer, err := opt.ContentStore.ReaderAt(ctx, desc) - if err != nil { - return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err) - } - - // Unpack the tarfile to the rootfs path. - // FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions - if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{ - NoLchown: false, - ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted' - }); err != nil { - return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err) - } - } - - _, _ = bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest}) - - return nil -} diff --git a/pkg/helpers/imgworker/workeropts.go b/pkg/helpers/imgworker/workeropts.go deleted file mode 100644 index da126d34..00000000 --- a/pkg/helpers/imgworker/workeropts.go +++ /dev/null @@ -1,106 +0,0 @@ -package imgworker - -// FROM Slightly adapted from genuinetools/img worker - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/containerd/containerd/content/local" - "github.com/containerd/containerd/diff/apply" - "github.com/containerd/containerd/diff/walking" - ctdmetadata "github.com/containerd/containerd/metadata" - "github.com/containerd/containerd/platforms" - "github.com/containerd/containerd/remotes/docker" - ctdsnapshot "github.com/containerd/containerd/snapshots" - "github.com/containerd/containerd/snapshots/native" - "github.com/moby/buildkit/cache/metadata" - containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" - "github.com/moby/buildkit/util/binfmt_misc" - "github.com/moby/buildkit/util/leaseutil" - "github.com/moby/buildkit/worker/base" - specs "github.com/opencontainers/image-spec/specs-go/v1" - bolt "go.etcd.io/bbolt" -) - -// createWorkerOpt creates a base.WorkerOpt to be used for a new worker. -func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) { - - if c.opts != nil { - return *c.opts, nil - } - - // Create the metadata store. - md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db")) - if err != nil { - return opt, err - } - - snapshotRoot := filepath.Join(c.root, "snapshots") - - s, err := native.NewSnapshotter(snapshotRoot) - if err != nil { - return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err) - } - - // Create the content store locally. - contentStore, err := local.NewStore(filepath.Join(c.root, "content")) - if err != nil { - return opt, err - } - - // Open the bolt database for metadata. - db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil) - if err != nil { - return opt, err - } - - // Create the new database for metadata. - mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{ - c.backend: s, - }) - if err := mdb.Init(context.TODO()); err != nil { - return opt, err - } - - // Create the image store. - imageStore := ctdmetadata.NewImageStore(mdb) - - contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit") - - id, err := base.ID(c.root) - if err != nil { - return opt, err - } - - xlabels := base.Labels("oci", c.backend) - - var supportedPlatforms []specs.Platform - for _, p := range binfmt_misc.SupportedPlatforms(false) { - parsed, err := platforms.Parse(p) - if err != nil { - return opt, err - } - supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed)) - } - - opt = base.WorkerOpt{ - ID: id, - Labels: xlabels, - MetadataStore: md, - Snapshotter: containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil), - ContentStore: contentStore, - Applier: apply.NewFileSystemApplier(contentStore), - Differ: walking.NewWalkingDiff(contentStore), - ImageStore: imageStore, - Platforms: supportedPlatforms, - RegistryHosts: docker.ConfigureDefaultRegistries(), - LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"), - GarbageCollect: mdb.GarbageCollect, - } - - c.opts = &opt - - return opt, err -} diff --git a/pkg/installer/client/docker.go b/pkg/installer/client/docker.go index 2dfa853c..89b076f0 100644 --- a/pkg/installer/client/docker.go +++ b/pkg/installer/client/docker.go @@ -109,7 +109,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact. } Info(fmt.Sprintf("Pulled: %s", info.Target.Digest)) - Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize)))) + Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size)))) Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile) newart := a @@ -170,7 +170,7 @@ func (c *DockerClient) DownloadFile(name string) (string, error) { } Info(fmt.Sprintf("Pulled: %s", info.Target.Digest)) - Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize)))) + Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size)))) Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name()) err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())