mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-07-21 02:40:57 +00:00
Merge pull request #11244 from katexochen/p/guest-pull-config
runtime: add option to force guest pull
This commit is contained in:
commit
ac934e001e
@ -28,6 +28,7 @@ There are several kinds of Kata configurations and they are listed below.
|
|||||||
| `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
|
| `io.katacontainers.config.runtime.sandbox_cgroup_only`| `boolean` | determines if Kata processes are managed only in sandbox cgroup |
|
||||||
| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
|
| `io.katacontainers.config.runtime.enable_pprof` | `boolean` | enables Golang `pprof` for `containerd-shim-kata-v2` process |
|
||||||
| `io.katacontainers.config.runtime.create_container_timeout` | `uint64` | the timeout for create a container in `seconds`, default is `60` |
|
| `io.katacontainers.config.runtime.create_container_timeout` | `uint64` | the timeout for create a container in `seconds`, default is `60` |
|
||||||
|
| `io.katacontainers.config.runtime.experimental_force_guest_pull` | `boolean` | forces the runtime to pull the image in the guest VM, default is `false`. This is an experimental feature and might be removed in the future. |
|
||||||
|
|
||||||
## Agent Options
|
## Agent Options
|
||||||
| Key | Value Type | Comments |
|
| Key | Value Type | Comments |
|
||||||
|
@ -175,6 +175,7 @@ DEFMSIZE9P := 8192
|
|||||||
DEFVFIOMODE := guest-kernel
|
DEFVFIOMODE := guest-kernel
|
||||||
DEFBINDMOUNTS := []
|
DEFBINDMOUNTS := []
|
||||||
DEFDANCONF := /run/kata-containers/dans
|
DEFDANCONF := /run/kata-containers/dans
|
||||||
|
DEFFORCEGUESTPULL := false
|
||||||
SED = sed
|
SED = sed
|
||||||
CLI_DIR = cmd
|
CLI_DIR = cmd
|
||||||
SHIMV2 = containerd-shim-kata-v2
|
SHIMV2 = containerd-shim-kata-v2
|
||||||
@ -503,6 +504,7 @@ USER_VARS += KATA_INSTALL_GROUP
|
|||||||
USER_VARS += KATA_INSTALL_OWNER
|
USER_VARS += KATA_INSTALL_OWNER
|
||||||
USER_VARS += KATA_INSTALL_CFG_PERMS
|
USER_VARS += KATA_INSTALL_CFG_PERMS
|
||||||
USER_VARS += DEFDANCONF
|
USER_VARS += DEFDANCONF
|
||||||
|
USER_VARS += DEFFORCEGUESTPULL
|
||||||
|
|
||||||
SOURCES := \
|
SOURCES := \
|
||||||
$(shell find . 2>&1 | grep -E '.*\.rs$$') \
|
$(shell find . 2>&1 | grep -E '.*\.rs$$') \
|
||||||
|
@ -280,6 +280,8 @@ DEFCREATECONTAINERTIMEOUT ?= 60
|
|||||||
# Default directory of directly attachable network config.
|
# Default directory of directly attachable network config.
|
||||||
DEFDANCONF := /run/kata-containers/dans
|
DEFDANCONF := /run/kata-containers/dans
|
||||||
|
|
||||||
|
DEFFORCEGUESTPULL := false
|
||||||
|
|
||||||
SED = sed
|
SED = sed
|
||||||
|
|
||||||
CLI_DIR = cmd
|
CLI_DIR = cmd
|
||||||
@ -753,6 +755,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_TEE
|
|||||||
USER_VARS += DEFBINDMOUNTS
|
USER_VARS += DEFBINDMOUNTS
|
||||||
USER_VARS += DEFCREATECONTAINERTIMEOUT
|
USER_VARS += DEFCREATECONTAINERTIMEOUT
|
||||||
USER_VARS += DEFDANCONF
|
USER_VARS += DEFDANCONF
|
||||||
|
USER_VARS += DEFFORCEGUESTPULL
|
||||||
USER_VARS += DEFVFIOMODE
|
USER_VARS += DEFVFIOMODE
|
||||||
USER_VARS += DEFVFIOMODE_SE
|
USER_VARS += DEFVFIOMODE_SE
|
||||||
USER_VARS += BUILDFLAGS
|
USER_VARS += BUILDFLAGS
|
||||||
|
@ -713,3 +713,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -698,3 +698,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -682,3 +682,8 @@ create_container_timeout = @DEFAULTTIMEOUT_NV@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -673,3 +673,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -639,3 +639,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -691,3 +691,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -676,3 +676,8 @@ create_container_timeout = @DEFCREATECONTAINERTIMEOUT@
|
|||||||
# to the hypervisor.
|
# to the hypervisor.
|
||||||
# (default: /run/kata-containers/dans)
|
# (default: /run/kata-containers/dans)
|
||||||
dan_conf = "@DEFDANCONF@"
|
dan_conf = "@DEFDANCONF@"
|
||||||
|
|
||||||
|
# Enforce guest pull. This instructs the runtime to communicate to the agent via annotations that
|
||||||
|
# the container image should be pulled in the guest, without using an external snapshotter.
|
||||||
|
# This is an experimental feature and might be removed in the future.
|
||||||
|
experimental_force_guest_pull = @DEFFORCEGUESTPULL@
|
||||||
|
@ -193,6 +193,7 @@ type runtime struct {
|
|||||||
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
|
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
|
||||||
CreateContainerTimeout uint64 `toml:"create_container_timeout"`
|
CreateContainerTimeout uint64 `toml:"create_container_timeout"`
|
||||||
DanConf string `toml:"dan_conf"`
|
DanConf string `toml:"dan_conf"`
|
||||||
|
ForceGuestPull bool `toml:"experimental_force_guest_pull"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type agent struct {
|
type agent struct {
|
||||||
@ -1587,6 +1588,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
|
|||||||
return "", config, err
|
return "", config, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
config.ForceGuestPull = tomlConf.Runtime.ForceGuestPull
|
||||||
|
|
||||||
return resolved, config, nil
|
return resolved, config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -171,6 +171,9 @@ type RuntimeConfig struct {
|
|||||||
|
|
||||||
// Base directory of directly attachable network config
|
// Base directory of directly attachable network config
|
||||||
DanConfig string
|
DanConfig string
|
||||||
|
|
||||||
|
// ForceGuestPull enforces guest pull independent of snapshotter annotations.
|
||||||
|
ForceGuestPull bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddKernelParam allows the addition of new kernel parameters to an existing
|
// AddKernelParam allows the addition of new kernel parameters to an existing
|
||||||
@ -1000,6 +1003,12 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := newAnnotationConfiguration(ocispec, vcAnnotations.ForceGuestPull).setBool(func(forceGuestPull bool) {
|
||||||
|
sbConfig.ForceGuestPull = forceGuestPull
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := newAnnotationConfiguration(ocispec, vcAnnotations.EnableVCPUsPinning).setBool(func(enableVCPUsPinning bool) {
|
if err := newAnnotationConfiguration(ocispec, vcAnnotations.EnableVCPUsPinning).setBool(func(enableVCPUsPinning bool) {
|
||||||
sbConfig.EnableVCPUsPinning = enableVCPUsPinning
|
sbConfig.EnableVCPUsPinning = enableVCPUsPinning
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@ -1145,6 +1154,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
|
|||||||
Experimental: runtime.Experimental,
|
Experimental: runtime.Experimental,
|
||||||
|
|
||||||
CreateContainerTimeout: runtime.CreateContainerTimeout,
|
CreateContainerTimeout: runtime.CreateContainerTimeout,
|
||||||
|
|
||||||
|
ForceGuestPull: runtime.ForceGuestPull,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {
|
if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil {
|
||||||
|
@ -93,7 +93,7 @@ type FilesystemShare struct {
|
|||||||
prepared bool
|
prepared bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFilesystemShare(s *Sandbox) (FilesystemSharer, error) {
|
func NewFilesystemShare(s *Sandbox) (*FilesystemShare, error) {
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := fsnotify.NewWatcher()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Creating watcher returned error %w", err)
|
return nil, fmt.Errorf("Creating watcher returned error %w", err)
|
||||||
@ -594,8 +594,29 @@ func (f *FilesystemShare) shareRootFilesystemWithErofs(ctx context.Context, c *C
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func forceGuestPull(c *Container) (*SharedFile, error) {
|
||||||
|
sf := &SharedFile{
|
||||||
|
guestPath: filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix),
|
||||||
|
}
|
||||||
|
guestPullVolume := &types.KataVirtualVolume{
|
||||||
|
VolumeType: types.KataVirtualVolumeImageGuestPullType,
|
||||||
|
ImagePull: &types.ImagePullVolume{
|
||||||
|
Metadata: map[string]string{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
vol, err := handleVirtualVolumeStorageObject(c, "", guestPullVolume)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("forcing guest pull virtual volume: %w", err)
|
||||||
|
}
|
||||||
|
sf.containerStorages = append(sf.containerStorages, vol)
|
||||||
|
return sf, nil
|
||||||
|
}
|
||||||
|
|
||||||
// func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) {
|
// func (c *Container) shareRootfs(ctx context.Context) (*grpc.Storage, string, error) {
|
||||||
func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) {
|
func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) (*SharedFile, error) {
|
||||||
|
if f.sandbox.IsGuestPullForced() {
|
||||||
|
return forceGuestPull(c)
|
||||||
|
}
|
||||||
|
|
||||||
if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
|
if HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
|
||||||
return f.shareRootFilesystemWithVirtualVolume(ctx, c)
|
return f.shareRootFilesystemWithVirtualVolume(ctx, c)
|
||||||
|
@ -14,6 +14,8 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
|
||||||
|
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -96,3 +98,87 @@ func TestSandboxSharedFilesystem(t *testing.T) {
|
|||||||
err = sandbox.fsShare.Cleanup(sandbox.ctx)
|
err = sandbox.fsShare.Cleanup(sandbox.ctx)
|
||||||
assert.NoError(err)
|
assert.NoError(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestShareRootFilesystem(t *testing.T) {
|
||||||
|
requireNewFilesystemShare := func(sandbox *Sandbox) *FilesystemShare {
|
||||||
|
fsShare, err := NewFilesystemShare(sandbox)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
return fsShare
|
||||||
|
}
|
||||||
|
|
||||||
|
testCases := map[string]struct {
|
||||||
|
fsSharer *FilesystemShare
|
||||||
|
container *Container
|
||||||
|
wantErr bool
|
||||||
|
wantSharedFile *SharedFile
|
||||||
|
}{
|
||||||
|
"force guest pull successful": {
|
||||||
|
fsSharer: requireNewFilesystemShare(&Sandbox{
|
||||||
|
config: &SandboxConfig{
|
||||||
|
ForceGuestPull: true,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
container: &Container{
|
||||||
|
id: "container-id-abc",
|
||||||
|
rootfsSuffix: "test-suffix",
|
||||||
|
config: &ContainerConfig{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"io.kubernetes.cri.image-name": "test-image-name",
|
||||||
|
},
|
||||||
|
CustomSpec: &specs.Spec{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"io.kubernetes.cri.container-type": "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantSharedFile: &SharedFile{
|
||||||
|
containerStorages: []*grpc.Storage{{
|
||||||
|
Fstype: "overlay",
|
||||||
|
Source: "test-image-name",
|
||||||
|
MountPoint: "/run/kata-containers/container-id-abc/test-suffix",
|
||||||
|
Driver: "image_guest_pull",
|
||||||
|
DriverOptions: []string{
|
||||||
|
"image_guest_pull={\"metadata\":{\"io.kubernetes.cri.image-name\":\"test-image-name\"}}",
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
guestPath: "/run/kata-containers/container-id-abc/test-suffix",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"force guest pull image name missing": {
|
||||||
|
fsSharer: requireNewFilesystemShare(&Sandbox{
|
||||||
|
config: &SandboxConfig{
|
||||||
|
ForceGuestPull: true,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
container: &Container{
|
||||||
|
id: "container-id-abc",
|
||||||
|
rootfsSuffix: "test-suffix",
|
||||||
|
config: &ContainerConfig{
|
||||||
|
Annotations: map[string]string{},
|
||||||
|
CustomSpec: &specs.Spec{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"io.kubernetes.cri.container-type": "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range testCases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
sharedFile, err := tc.fsSharer.ShareRootFilesystem(context.Background(), tc.container)
|
||||||
|
if tc.wantErr {
|
||||||
|
assert.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
assert.Equal(tc.wantSharedFile, sharedFile)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -283,6 +283,9 @@ const (
|
|||||||
|
|
||||||
// CreateContainerTimeout is a sandbox annotaion that sets the create container timeout.
|
// CreateContainerTimeout is a sandbox annotaion that sets the create container timeout.
|
||||||
CreateContainerTimeout = kataAnnotRuntimePrefix + "create_container_timeout"
|
CreateContainerTimeout = kataAnnotRuntimePrefix + "create_container_timeout"
|
||||||
|
|
||||||
|
// ForceGuestPull is a sandbox annotation that sets experimental_force_guest_pull.
|
||||||
|
ForceGuestPull = kataAnnotRuntimePrefix + "experimental_force_guest_pull"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Agent related annotations
|
// Agent related annotations
|
||||||
|
@ -186,6 +186,9 @@ type SandboxConfig struct {
|
|||||||
// Create container timeout which, if provided, indicates the create container timeout
|
// Create container timeout which, if provided, indicates the create container timeout
|
||||||
// needed for the workload(s)
|
// needed for the workload(s)
|
||||||
CreateContainerTimeout uint64
|
CreateContainerTimeout uint64
|
||||||
|
|
||||||
|
// ForceGuestPull enforces guest pull independent of snapshotter annotations.
|
||||||
|
ForceGuestPull bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// valid checks that the sandbox configuration is valid.
|
// valid checks that the sandbox configuration is valid.
|
||||||
@ -448,6 +451,14 @@ func (s *Sandbox) IOStream(containerID, processID string) (io.WriteCloser, io.Re
|
|||||||
return c.ioStream(processID)
|
return c.ioStream(processID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsGuestPullEnforced returns true if guest pull is forced through the sandbox configuration.
|
||||||
|
func (s *Sandbox) IsGuestPullForced() bool {
|
||||||
|
if s.config == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return s.config.ForceGuestPull
|
||||||
|
}
|
||||||
|
|
||||||
func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
|
func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
|
||||||
span, _ := katatrace.Trace(ctx, nil, "createAssets", sandboxTracingTags, map[string]string{"sandbox_id": sandboxConfig.ID})
|
span, _ := katatrace.Trace(ctx, nil, "createAssets", sandboxTracingTags, map[string]string{"sandbox_id": sandboxConfig.ID})
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
Loading…
Reference in New Issue
Block a user