Merge pull request #1649 from mcastelino/topic/jail

Firecracker Add jailer support for firecracker

Makefile

@@ -116,6 +116,7 @@ CONFIG_FILE = configuration.toml
 
 HYPERVISOR_ACRN = acrn
 HYPERVISOR_FC = firecracker
+JAILER_FC = jailer
 HYPERVISOR_NEMU = nemu
 HYPERVISOR_QEMU = qemu
 
@@ -130,6 +131,7 @@ QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
 NEMUPATH := $(NEMUBINDIR)/$(NEMUCMD)
 
 FCPATH = $(FCBINDIR)/$(FCCMD)
+FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD)
 
 ACRNPATH := $(ACRNBINDIR)/$(ACRNCMD)
 ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD)
@@ -355,6 +357,7 @@ USER_VARS += ACRNPATH
 USER_VARS += ACRNCTLPATH
 USER_VARS += FCCMD
 USER_VARS += FCPATH
+USER_VARS += FCJAILERPATH
 USER_VARS += NEMUCMD
 USER_VARS += NEMUPATH
 USER_VARS += SYSCONFIG
@@ -516,6 +519,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \
 		-e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \
 		-e "s|@FCPATH@|$(FCPATH)|g" \
+		-e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \
 		-e "s|@NEMUPATH@|$(NEMUPATH)|g" \
 		-e "s|@ACRNPATH@|$(ACRNPATH)|g" \
 		-e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \

arch/amd64-options.mk

@@ -13,6 +13,8 @@ QEMUCMD := qemu-system-x86_64
 
 # Firecracker binary name
 FCCMD := firecracker
+# Firecracker's jailer binary name
+FCJAILERCMD := jailer
 
 # NEMU binary name
 NEMUCMD := nemu-system-x86_64

cli/config/configuration-fc.toml.in

@@ -12,6 +12,11 @@
 
 [hypervisor.firecracker]
 path = "@FCPATH@"
+# Path for the jailer specific to firecracker
+# If the jailer path is not set kata will launch firecracker
+# without a jail. If the jailer is set firecracker will be
+# launched in a jailed enviornment created by the jailer
+jailer_path = "@FCJAILERPATH@"
 kernel = "@KERNELPATH_FC@"
 image = "@IMAGEPATH@"
 

pkg/katautils/config-settings.go

@@ -10,6 +10,7 @@ package katautils
 
 var defaultHypervisorPath = "/usr/bin/qemu-lite-system-x86_64"
 var defaultHypervisorCtlPath = "/usr/bin/acrnctl"
+var defaultJailerPath = "/usr/bin/jailer"
 var defaultImagePath = "/usr/share/kata-containers/kata-containers.img"
 var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container"
 var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"

pkg/katautils/config.go

@@ -84,6 +84,7 @@ type factory struct {
 
 type hypervisor struct {
 	Path                    string `toml:"path"`
+	JailerPath              string `toml:"jailer_path"`
 	Kernel                  string `toml:"kernel"`
 	CtlPath                 string `toml:"ctlpath"`
 	Initrd                  string `toml:"initrd"`
@@ -175,6 +176,16 @@ func (h hypervisor) ctlpath() (string, error) {
 	return ResolvePath(p)
 }
 
+func (h hypervisor) jailerPath() (string, error) {
+	p := h.JailerPath
+
+	if h.JailerPath == "" {
+		return "", nil
+	}
+
+	return ResolvePath(p)
+}
+
 func (h hypervisor) kernel() (string, error) {
 	p := h.Kernel
 
@@ -463,6 +474,11 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		return vc.HypervisorConfig{}, err
 	}
 
+	jailer, err := h.jailerPath()
+	if err != nil {
+		return vc.HypervisorConfig{}, err
+	}
+
 	kernel, err := h.kernel()
 	if err != nil {
 		return vc.HypervisorConfig{}, err
@@ -491,6 +507,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 
 	return vc.HypervisorConfig{
 		HypervisorPath:        hypervisor,
+		JailerPath:            jailer,
 		KernelPath:            kernel,
 		InitrdPath:            initrd,
 		ImagePath:             image,
@@ -915,6 +932,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
 func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 	return vc.HypervisorConfig{
 		HypervisorPath:          defaultHypervisorPath,
+		JailerPath:              defaultJailerPath,
 		KernelPath:              defaultKernelPath,
 		ImagePath:               defaultImagePath,
 		InitrdPath:              defaultInitrdPath,

pkg/katautils/config_test.go

@@ -514,6 +514,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 	proxyPath := path.Join(dir, "proxy")
 	hypervisorPath := path.Join(dir, "hypervisor")
 	defaultHypervisorPath = hypervisorPath
+	jailerPath := path.Join(dir, "jailer")
+	defaultJailerPath = jailerPath
 	netmonPath := path.Join(dir, "netmon")
 
 	imagePath := path.Join(dir, "image.img")
@@ -524,12 +526,14 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 	savedDefaultImagePath := defaultImagePath
 	savedDefaultInitrdPath := defaultInitrdPath
 	savedDefaultHypervisorPath := defaultHypervisorPath
+	savedDefaultJailerPath := defaultJailerPath
 	savedDefaultKernelPath := defaultKernelPath
 
 	defer func() {
 		defaultImagePath = savedDefaultImagePath
 		defaultInitrdPath = savedDefaultInitrdPath
 		defaultHypervisorPath = savedDefaultHypervisorPath
+		defaultJailerPath = savedDefaultJailerPath
 		defaultKernelPath = savedDefaultKernelPath
 	}()
 
@@ -538,9 +542,10 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 	defaultImagePath = imagePath
 	defaultInitrdPath = initrdPath
 	defaultHypervisorPath = hypervisorPath
+	defaultJailerPath = jailerPath
 	defaultKernelPath = kernelPath
 
-	for _, file := range []string{defaultImagePath, defaultInitrdPath, defaultHypervisorPath, defaultKernelPath} {
+	for _, file := range []string{defaultImagePath, defaultInitrdPath, defaultHypervisorPath, defaultJailerPath, defaultKernelPath} {
 		err = WriteFile(file, "foo", testFileMode)
 		if err != nil {
 			t.Fatal(err)
@@ -588,6 +593,11 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		t.Error(err)
 	}
 
+	err = createEmptyFile(jailerPath)
+	if err != nil {
+		t.Error(err)
+	}
+
 	err = createEmptyFile(netmonPath)
 	if err != nil {
 		t.Error(err)
@@ -600,6 +610,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 
 	expectedHypervisorConfig := vc.HypervisorConfig{
 		HypervisorPath:        defaultHypervisorPath,
+		JailerPath:            defaultJailerPath,
 		KernelPath:            defaultKernelPath,
 		ImagePath:             defaultImagePath,
 		InitrdPath:            defaultInitrdPath,

virtcontainers/acrn.go

@@ -278,7 +278,7 @@ func (a *acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) {
 }
 
 // createSandbox is the Hypervisor sandbox creation.
-func (a *acrn) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
+func (a *acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
 	// Save the tracing context
 	a.ctx = ctx
 

virtcontainers/acrn_test.go

@@ -251,7 +251,7 @@ func TestAcrnCreateSandbox(t *testing.T) {
 		t.Fatalf("Could not create hypervisor file %s: %v", testAcrnPath, err)
 	}
 
-	if err := a.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+	if err := a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
 		t.Fatal(err)
 	}
 

virtcontainers/fc.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
@@ -45,7 +46,11 @@ const (
 const (
 	//fcTimeout is the maximum amount of time in seconds to wait for the VMM to respond
 	fcTimeout = 10
-	fireSocket           = "firecracker.sock"
+	fcSocket  = "api.socket"
+	//Name of the files within jailer root
+	//Having predefined names helps with cleanup
+	fcKernel             = "vmlinux"
+	fcRootfs             = "rootfs"
 	fcStopSandboxTimeout = 15
 	// This indicates the number of block devices that can be attached to the
 	// firecracker guest VM.
@@ -103,17 +108,26 @@ func (s *firecrackerState) set(state vmmState) {
 // firecracker is an Hypervisor interface implementation for the firecracker hypervisor.
 type firecracker struct {
 	id            string //Unique ID per pod. Normally maps to the sandbox id
-	state firecrackerState
+	vmPath        string //All jailed VM assets need to be under this
+	chrootBaseDir string //chroot base for the jailer
+	jailerRoot    string
+	socketPath    string
+	netNSPath     string
+	uid           string //UID and GID to be used for the VMM
+	gid           string
+
 	info FirecrackerInfo
 
 	firecrackerd *exec.Cmd           //Tracks the firecracker process itself
-	fcClient     *client.Firecracker //Tracks the current active connection
-	socketPath   string
+	connection   *client.Firecracker //Tracks the current active connection
 
 	store          *store.VCStore
+	ctx            context.Context
 	config         HypervisorConfig
 	pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready
-	ctx            context.Context
+
+	state  firecrackerState
+	jailed bool //Set to true if jailer is enabled
 }
 
 type firecrackerDevice struct {
@@ -140,9 +154,47 @@ func (fc *firecracker) trace(name string) (opentracing.Span, context.Context) {
 	return span, ctx
 }
 
+// bindMount bind mounts a source in to a destination. This will
+// do some bookkeeping:
+// * evaluate all symlinks
+// * ensure the source exists
+// * recursively create the destination
+func (fc *firecracker) bindMount(ctx context.Context, source, destination string, readonly bool) error {
+	span, _ := trace(ctx, "bindMount")
+	defer span.Finish()
+
+	if source == "" {
+		return fmt.Errorf("source must be specified")
+	}
+	if destination == "" {
+		return fmt.Errorf("destination must be specified")
+	}
+
+	absSource, err := filepath.EvalSymlinks(source)
+	if err != nil {
+		return fmt.Errorf("Could not resolve symlink for source %v", source)
+	}
+
+	if err := ensureDestinationExists(absSource, destination); err != nil {
+		return fmt.Errorf("Could not create destination mount point %v: %v", destination, err)
+	}
+
+	if err := syscall.Mount(absSource, destination, "bind", syscall.MS_BIND|syscall.MS_SLAVE, ""); err != nil {
+		return fmt.Errorf("Could not bind mount %v to %v: %v", absSource, destination, err)
+	}
+
+	// For readonly bind mounts, we need to remount with the readonly flag.
+	// This is needed as only very recent versions of libmount/util-linux support "bind,ro"
+	if readonly {
+		return syscall.Mount(absSource, destination, "bind", uintptr(syscall.MS_BIND|syscall.MS_SLAVE|syscall.MS_REMOUNT|syscall.MS_RDONLY), "")
+	}
+
+	return nil
+}
+
 // For firecracker this call only sets the internal structure up.
 // The sandbox will be created and started through startSandbox().
-func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error {
+func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error {
 	fc.ctx = ctx
 
 	span, _ := fc.trace("createSandbox")
@@ -151,10 +203,34 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, hypervisorC
 	//TODO: check validity of the hypervisor config provided
 	//https://github.com/kata-containers/runtime/issues/1065
 	fc.id = id
-	fc.socketPath = filepath.Join(store.SandboxRuntimeRootPath(fc.id), fireSocket)
 	fc.store = vcStore
-	fc.config = *hypervisorConfig
 	fc.state.set(notReady)
+	fc.config = *hypervisorConfig
+
+	// When running with jailer all resources need to be under
+	// a specific location and that location needs to have
+	// exec permission (i.e. should not be mounted noexec, e.g. /run, /var/run)
+	// Also unix domain socket names have a hard limit
+	// #define UNIX_PATH_MAX   108
+	// Keep it short and live within the jailer expected paths
+	// <chroot_base>/<exec_file_name>/<id>/
+	// Also jailer based on the id implicitly sets up cgroups under
+	// <cgroups_base>/<exec_file_name>/<id>/
+	hypervisorName := filepath.Base(hypervisorConfig.HypervisorPath)
+	//store.ConfigStoragePath cannot be used as we need exec perms
+	fc.chrootBaseDir = filepath.Join("/var/lib/", store.StoragePathSuffix)
+
+	fc.vmPath = filepath.Join(fc.chrootBaseDir, hypervisorName, fc.id)
+	fc.jailerRoot = filepath.Join(fc.vmPath, "root") // auto created by jailer
+	fc.socketPath = filepath.Join(fc.jailerRoot, fcSocket)
+
+	// So we need to repopulate this at startSandbox where it is valid
+	fc.netNSPath = networkNS.NetNsPath
+
+	// Till we create lower privileged kata user run as root
+	// https://github.com/kata-containers/runtime/issues/1869
+	fc.uid = "0"
+	fc.gid = "0"
 
 	// No need to return an error from there since there might be nothing
 	// to fetch if this is the first time the hypervisor is created.
@@ -239,9 +315,62 @@ func (fc *firecracker) fcInit(timeout int) error {
 	span, _ := fc.trace("fcInit")
 	defer span.Finish()
 
-	args := []string{"--api-sock", fc.socketPath}
+	if fc.config.JailerPath != "" {
+		fc.jailed = true
+	}
 
-	cmd := exec.Command(fc.config.HypervisorPath, args...)
+	// Fetch sandbox network to be able to access it from the sandbox structure.
+	var networkNS NetworkNamespace
+	if err := fc.store.Load(store.Network, &networkNS); err == nil {
+		if networkNS.NetNsPath == "" {
+			fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn()
+		}
+		fc.netNSPath = networkNS.NetNsPath
+	}
+
+	err := os.MkdirAll(fc.jailerRoot, store.DirMode)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			if err := os.RemoveAll(fc.vmPath); err != nil {
+				fc.Logger().WithError(err).Error("Fail to clean up vm directory")
+			}
+		}
+	}()
+
+	var args []string
+	var cmd *exec.Cmd
+
+	//https://github.com/firecracker-microvm/firecracker/blob/master/docs/jailer.md#jailer-usage
+	//--seccomp-level specifies whether seccomp filters should be installed and how restrictive they should be. Possible values are:
+	//0 : disabled.
+	//1 : basic filtering. This prohibits syscalls not whitelisted by Firecracker.
+	//2 (default): advanced filtering. This adds further checks on some of the parameters of the allowed syscalls.
+	if fc.jailed {
+		args = []string{
+			"--id", fc.id,
+			"--node", "0", //FIXME: Comprehend NUMA topology or explicit ignore
+			"--seccomp-level", "2",
+			"--exec-file", fc.config.HypervisorPath,
+			"--uid", "0", //https://github.com/kata-containers/runtime/issues/1869
+			"--gid", "0",
+			"--chroot-base-dir", fc.chrootBaseDir,
+			"--daemonize",
+		}
+		if fc.netNSPath != "" {
+			args = append(args, "--netns", fc.netNSPath)
+		}
+		cmd = exec.Command(fc.config.JailerPath, args...)
+	} else {
+		args = []string{"--api-sock", fc.socketPath}
+		cmd = exec.Command(fc.config.HypervisorPath, args...)
+
+	}
+
+	fc.Logger().WithField("hypervisor args", args).Debug()
+	fc.Logger().WithField("hypervisor cmd", cmd).Debug()
 	if err := cmd.Start(); err != nil {
 		fc.Logger().WithField("Error starting firecracker", err).Debug()
 		return err
@@ -249,7 +378,7 @@ func (fc *firecracker) fcInit(timeout int) error {
 
 	fc.info.PID = cmd.Process.Pid
 	fc.firecrackerd = cmd
-	fc.fcClient = fc.newFireClient()
+	fc.connection = fc.newFireClient()
 
 	if err := fc.waitVMM(timeout); err != nil {
 		fc.Logger().WithField("fcInit failed:", err).Debug()
@@ -314,11 +443,31 @@ func (fc *firecracker) client() *client.Firecracker {
 	span, _ := fc.trace("client")
 	defer span.Finish()
 
-	if fc.fcClient == nil {
-		fc.fcClient = fc.newFireClient()
+	if fc.connection == nil {
+		fc.connection = fc.newFireClient()
 	}
 
-	return fc.fcClient
+	return fc.connection
+}
+
+func (fc *firecracker) fcJailResource(src, dst string) (string, error) {
+	if src == "" || dst == "" {
+		return "", fmt.Errorf("fcJailResource: invalid jail locations: src:%v, dst:%v",
+			src, dst)
+	}
+	jailedLocation := filepath.Join(fc.jailerRoot, dst)
+	if err := fc.bindMount(context.Background(), src, jailedLocation, false); err != nil {
+		fc.Logger().WithField("bindMount failed", err).Error()
+		return "", err
+	}
+
+	if !fc.jailed {
+		return jailedLocation, nil
+	}
+
+	// This is the path within the jailed root
+	absPath := filepath.Join("/", dst)
+	return absPath, nil
 }
 
 func (fc *firecracker) fcSetBootSource(path, params string) error {
@@ -327,37 +476,47 @@ func (fc *firecracker) fcSetBootSource(path, params string) error {
 	fc.Logger().WithFields(logrus.Fields{"kernel-path": path,
 		"kernel-params": params}).Debug("fcSetBootSource")
 
+	kernelPath, err := fc.fcJailResource(path, fcKernel)
+	if err != nil {
+		return err
+	}
+
 	bootSrcParams := ops.NewPutGuestBootSourceParams()
 	src := &models.BootSource{
-		KernelImagePath: &path,
+		KernelImagePath: &kernelPath,
 		BootArgs:        params,
 	}
 	bootSrcParams.SetBody(src)
 
-	_, err := fc.client().Operations.PutGuestBootSource(bootSrcParams)
+	_, err = fc.client().Operations.PutGuestBootSource(bootSrcParams)
 	return err
 }
 
 func (fc *firecracker) fcSetVMRootfs(path string) error {
 	span, _ := fc.trace("fcSetVMRootfs")
 	defer span.Finish()
-	fc.Logger().WithField("VM-rootfs-path", path).Debug()
+
+	jailedRootfs, err := fc.fcJailResource(path, fcRootfs)
+	if err != nil {
+		return err
+	}
 
 	driveID := "rootfs"
 	driveParams := ops.NewPutGuestDriveByIDParams()
 	driveParams.SetDriveID(driveID)
 	isReadOnly := true
 	//Add it as a regular block device
-	//This allows us to use a paritioned root block device
+	//This allows us to use a partitoned root block device
 	isRootDevice := false
+	// This is the path within the jailed root
 	drive := &models.Drive{
 		DriveID:      &driveID,
 		IsReadOnly:   &isReadOnly,
 		IsRootDevice: &isRootDevice,
-		PathOnHost:   &path,
+		PathOnHost:   &jailedRootfs,
 	}
 	driveParams.SetBody(drive)
-	_, err := fc.client().Operations.PutGuestDriveByID(driveParams)
+	_, err = fc.client().Operations.PutGuestDriveByID(driveParams)
 	return err
 }
 
@@ -386,7 +545,7 @@ func (fc *firecracker) fcStartVM() error {
 
 	fc.Logger().Info("Starting VM")
 
-	fc.fcClient = fc.newFireClient()
+	fc.connection = fc.newFireClient()
 
 	actionParams := ops.NewCreateSyncActionParams()
 	actionType := "InstanceStart"
@@ -400,7 +559,6 @@ func (fc *firecracker) fcStartVM() error {
 	}
 
 	fc.state.set(vmReady)
-
 	return nil
 }
 
@@ -436,7 +594,6 @@ func (fc *firecracker) startSandbox(timeout int) error {
 	kernelParams := append(fc.config.KernelParams, fcKernelParams...)
 	strParams := SerializeParams(kernelParams, "=")
 	formattedParams := strings.Join(strParams, " ")
-
 	fc.fcSetBootSource(kernelPath, formattedParams)
 
 	image, err := fc.config.InitrdAssetPath()
@@ -483,7 +640,8 @@ func (fc *firecracker) createDiskPool() error {
 		isRootDevice := false
 
 		// Create a temporary file as a placeholder backend for the drive
-		hostURL, err := fc.store.Raw("")
+		//hostURL, err := fc.store.Raw("")
+		hostURL, err := fc.store.Raw(driveID)
 		if err != nil {
 			return err
 		}
@@ -494,11 +652,17 @@ func (fc *firecracker) createDiskPool() error {
 			return err
 		}
 
+		jailedDrive, err := fc.fcJailResource(u.Path, driveID)
+		if err != nil {
+			fc.Logger().WithField("createDiskPool failed", err).Error()
+			return err
+		}
+
 		drive := &models.Drive{
 			DriveID:      &driveID,
 			IsReadOnly:   &isReadOnly,
 			IsRootDevice: &isRootDevice,
-			PathOnHost:   &u.Path,
+			PathOnHost:   &jailedDrive,
 		}
 		driveParams.SetBody(drive)
 		_, err = fc.client().Operations.PutGuestDriveByID(driveParams)
@@ -510,6 +674,39 @@ func (fc *firecracker) createDiskPool() error {
 	return nil
 }
 
+func (fc *firecracker) umountResource(jailedPath string) {
+	hostPath := filepath.Join(fc.jailerRoot, jailedPath)
+	err := syscall.Unmount(hostPath, syscall.MNT_DETACH)
+	if err != nil {
+		fc.Logger().WithField("umountResource failed", err).Info()
+	}
+}
+
+// cleanup all jail artifacts
+func (fc *firecracker) cleanupJail() {
+	span, _ := fc.trace("cleanupJail")
+	defer span.Finish()
+
+	fc.umountResource(fcKernel)
+	fc.umountResource(fcRootfs)
+
+	for i := 0; i < fcDiskPoolSize; i++ {
+		fc.umountResource(fcDriveIndexToID(i))
+	}
+
+	//Run through the list second time as may have bindmounted
+	//to the same location twice. In the future this needs to
+	//be tracked so that we do not do this blindly
+	for i := 0; i < fcDiskPoolSize; i++ {
+		fc.umountResource(fcDriveIndexToID(i))
+	}
+
+	fc.Logger().WithField("cleaningJail", fc.vmPath).Info()
+	if err := os.RemoveAll(fc.vmPath); err != nil {
+		fc.Logger().WithField("cleanupJail failed", err).Error()
+	}
+}
+
 // stopSandbox will stop the Sandbox's VM.
 func (fc *firecracker) stopSandbox() (err error) {
 	span, _ := fc.trace("stopSandbox")
@@ -581,14 +778,21 @@ func (fc *firecracker) fcAddBlockDrive(drive config.BlockDrive) error {
 	driveParams.SetDriveID(driveID)
 	isReadOnly := false
 	isRootDevice := false
+
+	jailedDrive, err := fc.fcJailResource(drive.File, driveID)
+	if err != nil {
+		fc.Logger().WithField("fcAddBlockDrive failed", err).Error()
+		return err
+	}
 	driveFc := &models.Drive{
 		DriveID:      &driveID,
 		IsReadOnly:   &isReadOnly,
 		IsRootDevice: &isRootDevice,
-		PathOnHost:   &drive.File,
+		PathOnHost:   &jailedDrive,
 	}
+
 	driveParams.SetBody(driveFc)
-	_, err := fc.client().Operations.PutGuestDriveByID(driveParams)
+	_, err = fc.client().Operations.PutGuestDriveByID(driveParams)
 	return err
 }
 
@@ -603,13 +807,18 @@ func (fc *firecracker) fcUpdateBlockDrive(drive config.BlockDrive) error {
 	driveParams := ops.NewPatchGuestDriveByIDParams()
 	driveParams.SetDriveID(driveID)
 
+	jailedDrive, err := fc.fcJailResource(drive.File, driveID)
+	if err != nil {
+		fc.Logger().WithField("fcUpdateBlockDrive failed", err).Error()
+		return err
+	}
 	driveFc := &models.PartialDrive{
 		DriveID:    &driveID,
-		PathOnHost: &drive.File, //This is the only property that can be modified
+		PathOnHost: &jailedDrive, //This is the only property that can be modified
 	}
+
 	driveParams.SetBody(driveFc)
-	_, err := fc.client().Operations.PatchGuestDriveByID(driveParams)
-	if err != nil {
+	if _, err := fc.client().Operations.PatchGuestDriveByID(driveParams); err != nil {
 		return err
 	}
 
@@ -764,6 +973,7 @@ func (fc *firecracker) getThreadIDs() (vcpuThreadIDs, error) {
 }
 
 func (fc *firecracker) cleanup() error {
+	fc.cleanupJail()
 	return nil
 }
 

virtcontainers/hypervisor.go

@@ -225,6 +225,9 @@ type HypervisorConfig struct {
 	// HypervisorCtlPath is the hypervisor ctl executable host path.
 	HypervisorCtlPath string
 
+	// JailerPath is the jailer executable host path.
+	JailerPath string
+
 	// BlockDeviceDriver specifies the driver to be used for block device
 	// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
 	BlockDeviceDriver string
@@ -445,6 +448,8 @@ func (conf *HypervisorConfig) assetPath(t types.AssetType) (string, error) {
 		return conf.HypervisorPath, nil
 	case types.HypervisorCtlAsset:
 		return conf.HypervisorCtlPath, nil
+	case types.JailerAsset:
+		return conf.JailerPath, nil
 	case types.FirmwareAsset:
 		return conf.FirmwarePath, nil
 	default:
@@ -497,6 +502,11 @@ func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) {
 	return conf.assetPath(types.HypervisorCtlAsset)
 }
 
+// JailerAssetPath returns the VM Jailer path
+func (conf *HypervisorConfig) JailerAssetPath() (string, error) {
+	return conf.assetPath(types.JailerAsset)
+}
+
 // CustomHypervisorAsset returns true if the hypervisor asset is a custom one, false otherwise.
 func (conf *HypervisorConfig) CustomHypervisorAsset() bool {
 	return conf.isCustomAsset(types.HypervisorAsset)
@@ -640,7 +650,7 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) {
 // hypervisor is the virtcontainers hypervisor interface.
 // The default hypervisor implementation is Qemu.
 type hypervisor interface {
-	createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error
+	createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error
 	startSandbox(timeout int) error
 	stopSandbox() error
 	pauseSandbox() error

virtcontainers/mock_hypervisor.go

@@ -26,7 +26,7 @@ func (m *mockHypervisor) hypervisorConfig() HypervisorConfig {
 	return HypervisorConfig{}
 }
 
-func (m *mockHypervisor) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
+func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
 	err := hypervisorConfig.valid()
 	if err != nil {
 		return err

virtcontainers/mock_hypervisor_test.go

@@ -28,7 +28,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) {
 	ctx := context.Background()
 
 	// wrong config
-	if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err == nil {
+	if err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil); err == nil {
 		t.Fatal()
 	}
 
@@ -38,7 +38,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) {
 		HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor),
 	}
 
-	if err := m.createSandbox(ctx, sandbox.config.ID, &sandbox.config.HypervisorConfig, nil); err != nil {
+	if err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil); err != nil {
 		t.Fatal(err)
 	}
 }

virtcontainers/pkg/annotations/annotations.go

@@ -20,6 +20,9 @@ const (
 	// HypervisorPath is a sandbox annotation for passing a per container path pointing at the hypervisor that will run the container VM.
 	HypervisorPath = vcAnnotationsPrefix + "HypervisorPath"
 
+	// JailerPath is a sandbox annotation for passing a per container path pointing at the jailer that will constrain the container VM.
+	JailerPath = vcAnnotationsPrefix + "JailerPath"
+
 	// FirmwarePath is a sandbox annotation for passing a per container path pointing at the guest firmware that will run the container VM.
 	FirmwarePath = vcAnnotationsPrefix + "FirmwarePath"
 
@@ -35,6 +38,9 @@ const (
 	// HypervisorHash is an sandbox annotation for passing a container hypervisor binary SHA-512 hash value.
 	HypervisorHash = vcAnnotationsPrefix + "HypervisorHash"
 
+	// JailerHash is an sandbox annotation for passing a jailer binary SHA-512 hash value.
+	JailerHash = vcAnnotationsPrefix + "JailerHash"
+
 	// FirmwareHash is an sandbox annotation for passing a container guest firmware SHA-512 hash value.
 	FirmwareHash = vcAnnotationsPrefix + "FirmwareHash"
 

virtcontainers/qemu.go

@@ -447,7 +447,7 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo
 }
 
 // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
-func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
+func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error {
 	// Save the tracing context
 	q.ctx = ctx
 

virtcontainers/qemu_test.go

@@ -104,7 +104,7 @@ func TestQemuCreateSandbox(t *testing.T) {
 		t.Fatalf("Could not create parent directory %s: %v", parentDir, err)
 	}
 
-	if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+	if err := q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
 		t.Fatal(err)
 	}
 
@@ -148,7 +148,7 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+	if err := q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
 		t.Fatalf("Qemu createSandbox() is not expected to fail because of missing parent directory for storage: %v", err)
 	}
 }
@@ -505,7 +505,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	}
 	q := &qemu{}
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
-	if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+	if err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
 		t.Fatal(err)
 	}
 	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true)
@@ -522,7 +522,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS
 	sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir
 
-	err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store)
+	err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store)
 
 	expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work")
 	assert.Equal(expectErr, err)
@@ -534,7 +534,7 @@ func TestQemuFileBackedMem(t *testing.T) {
 	}
 	q = &qemu{}
 	sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc"
-	if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
+	if err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store); err != nil {
 		t.Fatal(err)
 	}
 	assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false)

virtcontainers/sandbox.go

@@ -574,7 +574,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		}
 	}()
 
-	if err = s.hypervisor.createSandbox(ctx, s.id, &sandboxConfig.HypervisorConfig, s.store); err != nil {
+	if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil {
 		return nil, err
 	}
 

virtcontainers/types/asset.go

@@ -29,6 +29,8 @@ func (t AssetType) Annotations() (string, string, error) {
 		return annotations.InitrdPath, annotations.InitrdHash, nil
 	case HypervisorAsset:
 		return annotations.HypervisorPath, annotations.HypervisorHash, nil
+	case JailerAsset:
+		return annotations.JailerPath, annotations.JailerHash, nil
 	case FirmwareAsset:
 		return annotations.FirmwarePath, annotations.FirmwareHash, nil
 	}
@@ -51,6 +53,8 @@ const (
 
 	// HypervisorCtlAsset is an hypervisor control asset.
 	HypervisorCtlAsset AssetType = "hypervisorctl"
+	// JailerAsset is an jailer asset.
+	JailerAsset AssetType = "jailer"
 
 	// FirmwareAsset is a firmware asset.
 	FirmwareAsset AssetType = "firmware"
@@ -88,6 +92,8 @@ func (a *Asset) Valid() bool {
 		return true
 	case HypervisorAsset:
 		return true
+	case JailerAsset:
+		return true
 	case FirmwareAsset:
 		return true
 	}

virtcontainers/vm.go

@@ -172,7 +172,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
 		}
 	}()
 
-	if err = hypervisor.createSandbox(ctx, id, &config.HypervisorConfig, vcStore); err != nil {
+	if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore); err != nil {
 		return nil, err
 	}