kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-07-31 06:50:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Drop imgworker client

Browse Source 
Cleanup code which allows to extract images in a privileged way. the
containerd way supersedes it.
This commit is contained in:
Ettore Di Giacinto 2021-10-14 22:05:30 +02:00
parent 9c19a7ec35
commit 6ea05e59ea
9 changed files with 11 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/util.go
View File

@ -84,7 +84,7 @@ func NewUnpackCommand() *cobra.Command {
os.Exit(1)
}
Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
},
}

49
pkg/helpers/docker/docker.go
View File

@ -24,7 +24,6 @@ import (
"github.com/containerd/containerd/images"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"github.com/mudler/luet/pkg/helpers/imgworker"
continerdarchive "github.com/containerd/containerd/archive"
"github.com/docker/cli/cli/trust"
@ -130,31 +129,6 @@ type UnpackEventData struct {
Dest string
}
// privilegedExtractImage uses the imgworker (which requires privileges) to extract a container image
func privilegedExtractImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
defer os.RemoveAll(temp)
c, err := imgworker.New(temp, auth)
if err != nil {
return nil, errors.Wrapf(err, "failed creating client")
}
defer c.Close()
listedImage, err := c.Pull(image)
if err != nil {
return nil, errors.Wrapf(err, "failed listing images")
}
os.RemoveAll(dest)
bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
err = c.Unpack(image, dest)
bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
return listedImage, err
}
// UnarchiveLayers extract layers with archive.Untar from docker instead of containerd
func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.AuthConfig, verify bool) (int64, error) {
layers, err := img.Layers()
@ -192,7 +166,7 @@ func UnarchiveLayers(temp string, img v1.Image, image, dest string, auth *types.
}
// DownloadAndExtractDockerImage extracts a container image natively. It supports privileged/unprivileged mode
func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*images.Image, error) {
if verify {
img, err := verifyImage(image, auth)
if err != nil {
@ -201,10 +175,6 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
image = img
}
if os.Getenv("LUET_PRIVILEGED_EXTRACT") == "true" {
return privilegedExtractImage(temp, image, dest, auth, verify)
}
if !fileHelper.Exists(dest) {
if err := os.MkdirAll(dest, os.ModePerm); err != nil {
return nil, errors.Wrapf(err, "cannot create destination directory")
@ -249,17 +219,14 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
return &imgworker.ListedImage{
Image: images.Image{
Name: image,
Labels: m.Annotations,
Target: specs.Descriptor{
MediaType: string(mt),
Digest: digest.Digest(d.String()),
Size: c,
},
return &images.Image{
Name: image,
Labels: m.Annotations,
Target: specs.Descriptor{
MediaType: string(mt),
Digest: digest.Digest(d.String()),
Size: c,
},
ContentSize: c,
}, nil
}

36
pkg/helpers/imgworker/auth.go
View File

@ -1,36 +0,0 @@
package imgworker
import (
"context"
"github.com/docker/docker/api/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"google.golang.org/grpc"
)
func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable {
return &authProvider{
config: auth,
}
}
type authProvider struct {
config *types.AuthConfig
}
func (ap *authProvider) Register(server *grpc.Server) {
// no-op
}
func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) {
res := &auth.CredentialsResponse{}
if ap.config.IdentityToken != "" {
res.Secret = ap.config.IdentityToken
} else {
res.Username = ap.config.Username
res.Secret = ap.config.Password
}
return res, nil
}

81
pkg/helpers/imgworker/client.go
View File

@ -1,81 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"os"
"path/filepath"
"github.com/containerd/containerd/namespaces"
dockertypes "github.com/docker/docker/api/types"
"github.com/genuinetools/img/types"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/appcontext"
"github.com/moby/buildkit/worker/base"
"github.com/pkg/errors"
)
// Client holds the information for the client we will use for communicating
// with the buildkit controller.
type Client struct {
backend string
localDirs map[string]string
root string
sessionManager *session.Manager
controller *control.Controller
opts *base.WorkerOpt
sess *session.Session
ctx context.Context
auth *dockertypes.AuthConfig
}
// New returns a new client for communicating with the buildkit controller.
func New(root string, auth *dockertypes.AuthConfig) (*Client, error) {
// Native backend is fine, our images have just one layer. No need to depend on anything
backend := types.NativeBackend
// Create the root/
root = filepath.Join(root, "runc", backend)
if err := os.MkdirAll(root, 0700); err != nil {
return nil, err
}
c := &Client{
backend: types.NativeBackend,
root: root,
localDirs: nil,
auth: auth,
}
if err := c.prepare(); err != nil {
return nil, errors.Wrapf(err, "failed preparing client")
}
// Create the start of the client.
return c, nil
}
func (c *Client) Close() {
c.sess.Close()
}
func (c *Client) prepare() error {
ctx := appcontext.Context()
sess, sessDialer, err := c.Session(ctx)
if err != nil {
return errors.Wrapf(err, "failed creating Session")
}
ctx = session.NewContext(ctx, sess.ID())
ctx = namespaces.WithNamespace(ctx, "buildkit")
c.ctx = ctx
c.sess = sess
go func() {
sess.Run(ctx, sessDialer)
}()
return nil
}

129
pkg/helpers/imgworker/pull.go
View File

@ -1,129 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"fmt"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
imageexporter "github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/source/containerimage"
)
// ListedImage represents an image structure returuned from ListImages.
// It extends containerd/images.Image with extra fields.
type ListedImage struct {
images.Image
ContentSize int64
}
// Pull retrieves an image from a remote registry.
func (c *Client) Pull(image string) (*ListedImage, error) {
ctx := c.ctx
sm, err := c.getSessionManager()
if err != nil {
return nil, err
}
// Parse the image name and tag.
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return nil, fmt.Errorf("parsing image name %q failed: %v", image, err)
}
// Add the latest lag if they did not provide one.
named = reference.TagNameOnly(named)
image = named.String()
// Get the identifier for the image.
identifier, err := source.NewImageIdentifier(image)
if err != nil {
return nil, err
}
// Create the worker opts.
opt, err := c.createWorkerOpt()
if err != nil {
return nil, fmt.Errorf("creating worker opt failed: %v", err)
}
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: opt.Snapshotter,
MetadataStore: opt.MetadataStore,
ContentStore: opt.ContentStore,
LeaseManager: opt.LeaseManager,
GarbageCollect: opt.GarbageCollect,
Applier: opt.Applier,
})
if err != nil {
return nil, err
}
// Create the source for the pull.
srcOpt := containerimage.SourceOpt{
Snapshotter: opt.Snapshotter,
ContentStore: opt.ContentStore,
Applier: opt.Applier,
CacheAccessor: cm,
ImageStore: opt.ImageStore,
RegistryHosts: opt.RegistryHosts,
LeaseManager: opt.LeaseManager,
}
src, err := containerimage.NewSource(srcOpt)
if err != nil {
return nil, err
}
s, err := src.Resolve(ctx, identifier, sm)
if err != nil {
return nil, err
}
ref, err := s.Snapshot(ctx)
if err != nil {
return nil, err
}
// Create the exporter for the pull.
iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
Snapshotter: opt.Snapshotter,
ContentStore: opt.ContentStore,
Differ: opt.Differ,
})
if err != nil {
return nil, err
}
expOpt := imageexporter.Opt{
SessionManager: sm,
ImageWriter: iw,
Images: opt.ImageStore,
RegistryHosts: opt.RegistryHosts,
LeaseManager: opt.LeaseManager,
}
exp, err := imageexporter.New(expOpt)
if err != nil {
return nil, err
}
e, err := exp.Resolve(ctx, map[string]string{"name": image})
if err != nil {
return nil, err
}
if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
return nil, err
}
// Get the image.
img, err := opt.ImageStore.Get(ctx, image)
if err != nil {
return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err)
}
size, err := img.Size(ctx, opt.ContentStore, platforms.Default())
if err != nil {
return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err)
}
return &ListedImage{Image: img, ContentSize: size}, nil
}

49
pkg/helpers/imgworker/session.go
View File

@ -1,49 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/session/testutil"
"github.com/pkg/errors"
)
func (c *Client) getSessionManager() (*session.Manager, error) {
if c.sessionManager == nil {
var err error
c.sessionManager, err = session.NewManager()
if err != nil {
return nil, err
}
}
return c.sessionManager, nil
}
// Session creates the session manager and returns the session and it's
// dialer.
func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) {
m, err := c.getSessionManager()
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create session manager")
}
sessionName := "luet"
s, err := session.NewSession(ctx, sessionName, "")
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create session")
}
syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs))
for name, d := range c.localDirs {
syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
}
s.Allow(filesync.NewFSSyncProvider(syncedDirs))
s.Allow(NewDockerAuthProvider(c.auth))
return s, sessionDialer(s, m), err
}
func sessionDialer(s *session.Session, m *session.Manager) session.Dialer {
// FIXME: rename testutil
return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
}

93
pkg/helpers/imgworker/unpack.go
View File

@ -1,93 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"errors"
"fmt"
"github.com/mudler/luet/pkg/bus"
"os"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
)
// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
// See how backends are unpacking images without asking for root permissions.
// UnpackEventData is the data structure to pass for the bus events
type UnpackEventData struct {
Image string
Dest string
}
// Unpack exports an image to a rootfs destination directory.
func (c *Client) Unpack(image, dest string) error {
ctx := c.ctx
if len(dest) < 1 {
return errors.New("destination directory for rootfs cannot be empty")
}
if _, err := os.Stat(dest); err == nil {
return fmt.Errorf("destination directory already exists: %s", dest)
}
// Parse the image name and tag.
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return fmt.Errorf("parsing image name %q failed: %v", image, err)
}
// Add the latest lag if they did not provide one.
named = reference.TagNameOnly(named)
image = named.String()
// Create the worker opts.
opt, err := c.createWorkerOpt()
if err != nil {
return fmt.Errorf("creating worker opt failed: %v", err)
}
if opt.ImageStore == nil {
return errors.New("image store is nil")
}
img, err := opt.ImageStore.Get(ctx, image)
if err != nil {
return fmt.Errorf("getting image %s from image store failed: %v", image, err)
}
manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default())
if err != nil {
return fmt.Errorf("getting image manifest failed: %v", err)
}
_,_ = bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
for _, desc := range manifest.Layers {
logrus.Debugf("Unpacking layer %s", desc.Digest.String())
// Read the blob from the content store.
layer, err := opt.ContentStore.ReaderAt(ctx, desc)
if err != nil {
return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err)
}
// Unpack the tarfile to the rootfs path.
// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
NoLchown: false,
ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
}); err != nil {
return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
}
}
_, _ = bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
return nil
}

106
pkg/helpers/imgworker/workeropts.go
View File

@ -1,106 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"fmt"
"path/filepath"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/diff/apply"
"github.com/containerd/containerd/diff/walking"
ctdmetadata "github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes/docker"
ctdsnapshot "github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/snapshots/native"
"github.com/moby/buildkit/cache/metadata"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/util/binfmt_misc"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/worker/base"
specs "github.com/opencontainers/image-spec/specs-go/v1"
bolt "go.etcd.io/bbolt"
)
// createWorkerOpt creates a base.WorkerOpt to be used for a new worker.
func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) {
if c.opts != nil {
return *c.opts, nil
}
// Create the metadata store.
md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db"))
if err != nil {
return opt, err
}
snapshotRoot := filepath.Join(c.root, "snapshots")
s, err := native.NewSnapshotter(snapshotRoot)
if err != nil {
return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err)
}
// Create the content store locally.
contentStore, err := local.NewStore(filepath.Join(c.root, "content"))
if err != nil {
return opt, err
}
// Open the bolt database for metadata.
db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil)
if err != nil {
return opt, err
}
// Create the new database for metadata.
mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{
c.backend: s,
})
if err := mdb.Init(context.TODO()); err != nil {
return opt, err
}
// Create the image store.
imageStore := ctdmetadata.NewImageStore(mdb)
contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
id, err := base.ID(c.root)
if err != nil {
return opt, err
}
xlabels := base.Labels("oci", c.backend)
var supportedPlatforms []specs.Platform
for _, p := range binfmt_misc.SupportedPlatforms(false) {
parsed, err := platforms.Parse(p)
if err != nil {
return opt, err
}
supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed))
}
opt = base.WorkerOpt{
ID: id,
Labels: xlabels,
MetadataStore: md,
Snapshotter: containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil),
ContentStore: contentStore,
Applier: apply.NewFileSystemApplier(contentStore),
Differ: walking.NewWalkingDiff(contentStore),
ImageStore: imageStore,
Platforms: supportedPlatforms,
RegistryHosts: docker.ConfigureDefaultRegistries(),
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
GarbageCollect: mdb.GarbageCollect,
}
c.opts = &opt
return opt, err
}

4
pkg/installer/client/docker.go
View File

@ -109,7 +109,7 @@ func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.
}
Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile)
newart := a
@ -170,7 +170,7 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
}
Info(fmt.Sprintf("Pulled: %s", info.Target.Digest))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize))))
Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
Debug("\nCopying file ", filepath.Join(temp, name), "to", file.Name())
err = fileHelper.CopyFile(filepath.Join(temp, name), file.Name())