kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-09-09 19:19:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Return a specific struct for the image

Browse Source 
And drop imgworker
This commit is contained in:
Ettore Di Giacinto
2021-06-04 11:09:12 +02:00
parent 796967cc9d
commit c3b552103f
8 changed files with 36 additions and 508 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/helpers/docker/docker.go
View File

@@ -20,9 +20,9 @@ import (
"encoding/hex"
"os"
"strings"
"time"
"github.com/containerd/containerd/archive"
"github.com/containerd/containerd/images"
"github.com/docker/cli/cli/trust"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
@@ -31,7 +31,6 @@ import (
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/mudler/luet/pkg/helpers/imgworker"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@@ -118,9 +117,33 @@ func (s staticAuth) Authorization() (*authn.AuthConfig, error) {
}, nil
}
type Image struct {
// Name of the image.
//
// To be pulled, it must be a reference compatible with resolvers.
//
// This field is required.
Name string
// Labels provide runtime decoration for the image record.
//
// There is no default behavior for how these labels are propagated. They
// only decorate the static metadata object.
//
// This field is optional.
Labels map[string]string
// Target describes the root content for this image. Typically, this is
// a manifest, index or manifest list.
Target specs.Descriptor
CreatedAt, UpdatedAt time.Time
ContentSize int64
}
// DownloadAndExtractDockerImage is a re-adaption
func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*imgworker.ListedImage, error) {
func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthConfig, verify bool) (*Image, error) {
if verify {
img, err := verifyImage(image, auth)
if err != nil {
@@ -167,8 +190,8 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
return nil, err
}
return &imgworker.ListedImage{
Image: images.Image{
return &Image{
Name: image,
Labels: m.Annotations,
Target: specs.Descriptor{
@@ -176,7 +199,7 @@ func DownloadAndExtractDockerImage(temp, image, dest string, auth *types.AuthCon
Digest: digest.Digest(d.String()),
Size: c,
},
},
ContentSize: c,
}, nil
}

36
pkg/helpers/imgworker/auth.go
View File

@@ -1,36 +0,0 @@
package imgworker
import (
"context"
"github.com/docker/docker/api/types"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"google.golang.org/grpc"
)
func NewDockerAuthProvider(auth *types.AuthConfig) session.Attachable {
return &authProvider{
config: auth,
}
}
type authProvider struct {
config *types.AuthConfig
}
func (ap *authProvider) Register(server *grpc.Server) {
// no-op
}
func (ap *authProvider) Credentials(ctx context.Context, req *auth.CredentialsRequest) (*auth.CredentialsResponse, error) {
res := &auth.CredentialsResponse{}
if ap.config.IdentityToken != "" {
res.Secret = ap.config.IdentityToken
} else {
res.Username = ap.config.Username
res.Secret = ap.config.Password
}
return res, nil
}

81
pkg/helpers/imgworker/client.go
View File

@@ -1,81 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"os"
"path/filepath"
"github.com/containerd/containerd/namespaces"
dockertypes "github.com/docker/docker/api/types"
"github.com/genuinetools/img/types"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/appcontext"
"github.com/moby/buildkit/worker/base"
"github.com/pkg/errors"
)
// Client holds the information for the client we will use for communicating
// with the buildkit controller.
type Client struct {
backend string
localDirs map[string]string
root string
sessionManager *session.Manager
controller *control.Controller
opts *base.WorkerOpt
sess *session.Session
ctx context.Context
auth *dockertypes.AuthConfig
}
// New returns a new client for communicating with the buildkit controller.
func New(root string, auth *dockertypes.AuthConfig) (*Client, error) {
// Native backend is fine, our images have just one layer. No need to depend on anything
backend := types.NativeBackend
// Create the root/
root = filepath.Join(root, "runc", backend)
if err := os.MkdirAll(root, 0700); err != nil {
return nil, err
}
c := &Client{
backend: types.NativeBackend,
root: root,
localDirs: nil,
auth: auth,
}
if err := c.prepare(); err != nil {
return nil, errors.Wrapf(err, "failed preparing client")
}
// Create the start of the client.
return c, nil
}
func (c *Client) Close() {
c.sess.Close()
}
func (c *Client) prepare() error {
ctx := appcontext.Context()
sess, sessDialer, err := c.Session(ctx)
if err != nil {
return errors.Wrapf(err, "failed creating Session")
}
ctx = session.NewContext(ctx, sess.ID())
ctx = namespaces.WithNamespace(ctx, "buildkit")
c.ctx = ctx
c.sess = sess
go func() {
sess.Run(ctx, sessDialer)
}()
return nil
}

129
pkg/helpers/imgworker/pull.go
View File

@@ -1,129 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"fmt"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
imageexporter "github.com/moby/buildkit/exporter/containerimage"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/source/containerimage"
)
// ListedImage represents an image structure returuned from ListImages.
// It extends containerd/images.Image with extra fields.
type ListedImage struct {
images.Image
ContentSize int64
}
// Pull retrieves an image from a remote registry.
func (c *Client) Pull(image string) (*ListedImage, error) {
ctx := c.ctx
sm, err := c.getSessionManager()
if err != nil {
return nil, err
}
// Parse the image name and tag.
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return nil, fmt.Errorf("parsing image name %q failed: %v", image, err)
}
// Add the latest lag if they did not provide one.
named = reference.TagNameOnly(named)
image = named.String()
// Get the identifier for the image.
identifier, err := source.NewImageIdentifier(image)
if err != nil {
return nil, err
}
// Create the worker opts.
opt, err := c.createWorkerOpt()
if err != nil {
return nil, fmt.Errorf("creating worker opt failed: %v", err)
}
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: opt.Snapshotter,
MetadataStore: opt.MetadataStore,
ContentStore: opt.ContentStore,
LeaseManager: opt.LeaseManager,
GarbageCollect: opt.GarbageCollect,
Applier: opt.Applier,
})
if err != nil {
return nil, err
}
// Create the source for the pull.
srcOpt := containerimage.SourceOpt{
Snapshotter: opt.Snapshotter,
ContentStore: opt.ContentStore,
Applier: opt.Applier,
CacheAccessor: cm,
ImageStore: opt.ImageStore,
RegistryHosts: opt.RegistryHosts,
LeaseManager: opt.LeaseManager,
}
src, err := containerimage.NewSource(srcOpt)
if err != nil {
return nil, err
}
s, err := src.Resolve(ctx, identifier, sm)
if err != nil {
return nil, err
}
ref, err := s.Snapshot(ctx)
if err != nil {
return nil, err
}
// Create the exporter for the pull.
iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{
Snapshotter: opt.Snapshotter,
ContentStore: opt.ContentStore,
Differ: opt.Differ,
})
if err != nil {
return nil, err
}
expOpt := imageexporter.Opt{
SessionManager: sm,
ImageWriter: iw,
Images: opt.ImageStore,
RegistryHosts: opt.RegistryHosts,
LeaseManager: opt.LeaseManager,
}
exp, err := imageexporter.New(expOpt)
if err != nil {
return nil, err
}
e, err := exp.Resolve(ctx, map[string]string{"name": image})
if err != nil {
return nil, err
}
if _, err := e.Export(ctx, exporter.Source{Ref: ref}); err != nil {
return nil, err
}
// Get the image.
img, err := opt.ImageStore.Get(ctx, image)
if err != nil {
return nil, fmt.Errorf("getting image %s from image store failed: %v", image, err)
}
size, err := img.Size(ctx, opt.ContentStore, platforms.Default())
if err != nil {
return nil, fmt.Errorf("calculating size of image %s failed: %v", img.Name, err)
}
return &ListedImage{Image: img, ContentSize: size}, nil
}

49
pkg/helpers/imgworker/session.go
View File

@@ -1,49 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/session/testutil"
"github.com/pkg/errors"
)
func (c *Client) getSessionManager() (*session.Manager, error) {
if c.sessionManager == nil {
var err error
c.sessionManager, err = session.NewManager()
if err != nil {
return nil, err
}
}
return c.sessionManager, nil
}
// Session creates the session manager and returns the session and it's
// dialer.
func (c *Client) Session(ctx context.Context) (*session.Session, session.Dialer, error) {
m, err := c.getSessionManager()
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create session manager")
}
sessionName := "luet"
s, err := session.NewSession(ctx, sessionName, "")
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create session")
}
syncedDirs := make([]filesync.SyncedDir, 0, len(c.localDirs))
for name, d := range c.localDirs {
syncedDirs = append(syncedDirs, filesync.SyncedDir{Name: name, Dir: d})
}
s.Allow(filesync.NewFSSyncProvider(syncedDirs))
s.Allow(NewDockerAuthProvider(c.auth))
return s, sessionDialer(s, m), err
}
func sessionDialer(s *session.Session, m *session.Manager) session.Dialer {
// FIXME: rename testutil
return session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))
}

93
pkg/helpers/imgworker/unpack.go
View File

@@ -1,93 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"errors"
"fmt"
"github.com/mudler/luet/pkg/bus"
"os"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
)
// TODO: this requires root permissions to mount/unmount layers, althrought it shouldn't be required.
// See how backends are unpacking images without asking for root permissions.
// UnpackEventData is the data structure to pass for the bus events
type UnpackEventData struct {
Image string
Dest string
}
// Unpack exports an image to a rootfs destination directory.
func (c *Client) Unpack(image, dest string) error {
ctx := c.ctx
if len(dest) < 1 {
return errors.New("destination directory for rootfs cannot be empty")
}
if _, err := os.Stat(dest); err == nil {
return fmt.Errorf("destination directory already exists: %s", dest)
}
// Parse the image name and tag.
named, err := reference.ParseNormalizedNamed(image)
if err != nil {
return fmt.Errorf("parsing image name %q failed: %v", image, err)
}
// Add the latest lag if they did not provide one.
named = reference.TagNameOnly(named)
image = named.String()
// Create the worker opts.
opt, err := c.createWorkerOpt()
if err != nil {
return fmt.Errorf("creating worker opt failed: %v", err)
}
if opt.ImageStore == nil {
return errors.New("image store is nil")
}
img, err := opt.ImageStore.Get(ctx, image)
if err != nil {
return fmt.Errorf("getting image %s from image store failed: %v", image, err)
}
manifest, err := images.Manifest(ctx, opt.ContentStore, img.Target, platforms.Default())
if err != nil {
return fmt.Errorf("getting image manifest failed: %v", err)
}
_,_ = bus.Manager.Publish(bus.EventImagePreUnPack, UnpackEventData{Image: image, Dest: dest})
for _, desc := range manifest.Layers {
logrus.Debugf("Unpacking layer %s", desc.Digest.String())
// Read the blob from the content store.
layer, err := opt.ContentStore.ReaderAt(ctx, desc)
if err != nil {
return fmt.Errorf("getting reader for digest %s failed: %v", desc.Digest.String(), err)
}
// Unpack the tarfile to the rootfs path.
// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
if err := archive.Untar(content.NewReader(layer), dest, &archive.TarOptions{
NoLchown: false,
ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted'
}); err != nil {
return fmt.Errorf("extracting tar for %s to directory %s failed: %v", desc.Digest.String(), dest, err)
}
}
_, _ = bus.Manager.Publish(bus.EventImagePostUnPack, UnpackEventData{Image: image, Dest: dest})
return nil
}

106
pkg/helpers/imgworker/workeropts.go
View File

@@ -1,106 +0,0 @@
package imgworker
// FROM Slightly adapted from genuinetools/img worker
import (
"context"
"fmt"
"path/filepath"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/diff/apply"
"github.com/containerd/containerd/diff/walking"
ctdmetadata "github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes/docker"
ctdsnapshot "github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/snapshots/native"
"github.com/moby/buildkit/cache/metadata"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/util/binfmt_misc"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/worker/base"
specs "github.com/opencontainers/image-spec/specs-go/v1"
bolt "go.etcd.io/bbolt"
)
// createWorkerOpt creates a base.WorkerOpt to be used for a new worker.
func (c *Client) createWorkerOpt() (opt base.WorkerOpt, err error) {
if c.opts != nil {
return *c.opts, nil
}
// Create the metadata store.
md, err := metadata.NewStore(filepath.Join(c.root, "metadata.db"))
if err != nil {
return opt, err
}
snapshotRoot := filepath.Join(c.root, "snapshots")
s, err := native.NewSnapshotter(snapshotRoot)
if err != nil {
return opt, fmt.Errorf("creating %s snapshotter failed: %v", c.backend, err)
}
// Create the content store locally.
contentStore, err := local.NewStore(filepath.Join(c.root, "content"))
if err != nil {
return opt, err
}
// Open the bolt database for metadata.
db, err := bolt.Open(filepath.Join(c.root, "containerdmeta.db"), 0644, nil)
if err != nil {
return opt, err
}
// Create the new database for metadata.
mdb := ctdmetadata.NewDB(db, contentStore, map[string]ctdsnapshot.Snapshotter{
c.backend: s,
})
if err := mdb.Init(context.TODO()); err != nil {
return opt, err
}
// Create the image store.
imageStore := ctdmetadata.NewImageStore(mdb)
contentStore = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
id, err := base.ID(c.root)
if err != nil {
return opt, err
}
xlabels := base.Labels("oci", c.backend)
var supportedPlatforms []specs.Platform
for _, p := range binfmt_misc.SupportedPlatforms(false) {
parsed, err := platforms.Parse(p)
if err != nil {
return opt, err
}
supportedPlatforms = append(supportedPlatforms, platforms.Normalize(parsed))
}
opt = base.WorkerOpt{
ID: id,
Labels: xlabels,
MetadataStore: md,
Snapshotter: containerdsnapshot.NewSnapshotter(c.backend, mdb.Snapshotter(c.backend), "buildkit", nil),
ContentStore: contentStore,
Applier: apply.NewFileSystemApplier(contentStore),
Differ: walking.NewWalkingDiff(contentStore),
ImageStore: imageStore,
Platforms: supportedPlatforms,
RegistryHosts: docker.ConfigureDefaultRegistries(),
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
GarbageCollect: mdb.GarbageCollect,
}
c.opts = &opt
return opt, err
}

3
pkg/installer/client/docker.go
View File

@@ -30,7 +30,6 @@ import (
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers/docker"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"github.com/mudler/luet/pkg/helpers/imgworker"
. "github.com/mudler/luet/pkg/logger"
)
@@ -141,7 +140,7 @@ func (c *DockerClient) DownloadFile(name string) (string, error) {
var file *os.File = nil
var err error
var temp, contentstore string
var info *imgworker.ListedImage
var info *docker.Image
// Files should be in URI/repository:<file>
ok := false