From c074ec4df16ad0d5adfbf37630fa5a20dae501a9 Mon Sep 17 00:00:00 2001 From: Yohei Ueda Date: Fri, 17 Jun 2022 23:40:19 +0900 Subject: [PATCH 1/4] runtime: Copy shared files recursively This patch enables recursive file copying when filesystem sharing is not used. Signed-off-by: Yohei Ueda Co-authored-by: stevenhorsman (cherry picked from commit 5422a056f2a7e0e796faed9739656f1a6aebe334) (cherry picked from commit 16055ce040bbd724be2916bc518d89b69c9e0ca5) Fixes: #7210 --- src/runtime/virtcontainers/fs_share_linux.go | 47 ++++++++++++----- src/runtime/virtcontainers/kata_agent.go | 51 ++++++++++++------- src/runtime/virtcontainers/kata_agent_test.go | 43 ++++++++++++++++ 3 files changed, 111 insertions(+), 30 deletions(-) diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index d2f9039726..cb5aff391e 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -11,6 +11,7 @@ import ( "context" "encoding/hex" "fmt" + "io/fs" "os" "path/filepath" "sync" @@ -240,23 +241,43 @@ func (f *FilesystemShare) ShareFile(ctx context.Context, c *Container, m *Mount) if !caps.IsFsSharingSupported() { f.Logger().Debug("filesystem sharing is not supported, files will be copied") - fileInfo, err := os.Stat(m.Source) - if err != nil { - return nil, err + var ignored bool + srcRoot := filepath.Clean(m.Source) + + walk := func(srcPath string, d fs.DirEntry, err error) error { + + if err != nil { + return err + } + + info, err := d.Info() + if err != nil { + return err + } + + if !(info.Mode().IsRegular() || info.Mode().IsDir() || (info.Mode()&os.ModeSymlink) == os.ModeSymlink) { + f.Logger().WithField("ignored-file", srcPath).Debug("Ignoring file as FS sharing not supported") + if srcPath == srcRoot { + // Ignore the mount if this is not a regular file (excludes socket, device, ...) as it cannot be handled by + // a simple copy. But this should not be treated as an error, only as a limitation. + ignored = true + return filepath.SkipDir + } + return nil + } + + dstPath := filepath.Join(guestPath, srcPath[len(srcRoot):]) + + return f.sandbox.agent.copyFile(ctx, srcPath, dstPath) } - // Ignore the mount if this is not a regular file (excludes - // directory, socket, device, ...) as it cannot be handled by - // a simple copy. But this should not be treated as an error, - // only as a limitation. - if !fileInfo.Mode().IsRegular() { - f.Logger().WithField("ignored-file", m.Source).Debug("Ignoring non-regular file as FS sharing not supported") + if err := filepath.WalkDir(srcRoot, walk); err != nil { + c.Logger().WithField("failed-file", m.Source).Debugf("failed to copy file to sandbox: %v", err) + return nil, err + } + if ignored { return nil, nil } - - if err := f.sandbox.agent.copyFile(ctx, m.Source, guestPath); err != nil { - return nil, err - } } else { // These mounts are created in the shared dir mountDest := filepath.Join(getMountPath(f.sandbox.ID()), filename) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 810deeee54..a214de5da6 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -2197,40 +2197,57 @@ func (k *kataAgent) setGuestDateTime(ctx context.Context, tv time.Time) error { func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error { var st unix.Stat_t - err := unix.Stat(src, &st) + err := unix.Lstat(src, &st) if err != nil { return fmt.Errorf("Could not get file %s information: %v", src, err) } - b, err := os.ReadFile(src) - if err != nil { - return fmt.Errorf("Could not read file %s: %v", src, err) + cpReq := &grpc.CopyFileRequest{ + Path: dst, + DirMode: uint32(DirMode), + FileMode: st.Mode, + Uid: int32(st.Uid), + Gid: int32(st.Gid), } - fileSize := int64(len(b)) + var b []byte + + switch sflag := st.Mode & unix.S_IFMT; sflag { + case unix.S_IFREG: + var err error + // TODO: Support incremental file copying instead of loading whole file into memory + b, err = os.ReadFile(src) + if err != nil { + return fmt.Errorf("Could not read file %s: %v", src, err) + } + cpReq.FileSize = int64(len(b)) + + case unix.S_IFDIR: + + case unix.S_IFLNK: + symlink, err := os.Readlink(src) + if err != nil { + return fmt.Errorf("Could not read symlink %s: %v", src, err) + } + cpReq.Data = []byte(symlink) + + default: + return fmt.Errorf("Unsupported file type: %o", sflag) + } k.Logger().WithFields(logrus.Fields{ "source": src, "dest": dst, }).Debugf("Copying file from host to guest") - cpReq := &grpc.CopyFileRequest{ - Path: dst, - DirMode: uint32(DirMode), - FileMode: uint32(st.Mode), - FileSize: fileSize, - Uid: int32(st.Uid), - Gid: int32(st.Gid), - } - // Handle the special case where the file is empty - if fileSize == 0 { - _, err = k.sendReq(ctx, cpReq) + if cpReq.FileSize == 0 { + _, err := k.sendReq(ctx, cpReq) return err } // Copy file by parts if it's needed - remainingBytes := fileSize + remainingBytes := cpReq.FileSize offset := int64(0) for remainingBytes > 0 { bytesToCopy := int64(len(b)) diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index c7fa059dcb..0bd4b3382c 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -993,6 +993,49 @@ func TestKataCopyFile(t *testing.T) { assert.NoError(err) } +func TestKataCopyFileWithSymlink(t *testing.T) { + assert := assert.New(t) + + url, err := mock.GenerateKataMockHybridVSock() + assert.NoError(err) + defer mock.RemoveKataMockHybridVSock(url) + + hybridVSockTTRPCMock := mock.HybridVSockTTRPCMock{} + err = hybridVSockTTRPCMock.Start(url) + assert.NoError(err) + defer hybridVSockTTRPCMock.Stop() + + k := &kataAgent{ + ctx: context.Background(), + state: KataAgentState{ + URL: url, + }, + } + + tempDir := t.TempDir() + + target := filepath.Join(tempDir, "target") + err = os.WriteFile(target, []byte("abcdefghi123456789"), 0666) + assert.NoError(err) + + symlink := filepath.Join(tempDir, "symlink") + os.Symlink(target, symlink) + + dst, err := os.CreateTemp("", "dst") + assert.NoError(err) + assert.NoError(dst.Close()) + defer os.Remove(dst.Name()) + + orgGrpcMaxDataSize := grpcMaxDataSize + grpcMaxDataSize = 1 + defer func() { + grpcMaxDataSize = orgGrpcMaxDataSize + }() + + err = k.copyFile(context.Background(), symlink, dst.Name()) + assert.NoError(err) +} + func TestKataCleanupSandbox(t *testing.T) { assert := assert.New(t) From ab13ef87eea31d7eb5a4c1080fa62e7e15f70f06 Mon Sep 17 00:00:00 2001 From: Pradipta Banerjee Date: Wed, 8 Feb 2023 06:58:11 +0000 Subject: [PATCH 2/4] runtime: propagate configmap/secrets etc changes for remote-hyp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For remote hypervisor, the configmap, secrets, downward-api or project-volumes are copied from host to guest. This patch watches for changes to the host files and copies the changes to the guest. Note that configmap updates takes significantly longer than updates via downward-api. This is similar across runc and Kata runtimes. Fixes: #7210 Signed-off-by: Pradipta Banerjee Signed-off-by: Julien Ropé (cherry picked from commit 3081cd5f8e847dbde90145bf20406953389707f9) (cherry picked from commit 68ec673bc4d9cd853eee51b21a0e91fcec149aad) --- src/runtime/virtcontainers/fs_share.go | 8 + src/runtime/virtcontainers/fs_share_darwin.go | 7 + src/runtime/virtcontainers/fs_share_linux.go | 308 +++++++++++++++++- src/runtime/virtcontainers/sandbox.go | 22 ++ 4 files changed, 342 insertions(+), 3 deletions(-) diff --git a/src/runtime/virtcontainers/fs_share.go b/src/runtime/virtcontainers/fs_share.go index fa8374c3b8..b5000291cc 100644 --- a/src/runtime/virtcontainers/fs_share.go +++ b/src/runtime/virtcontainers/fs_share.go @@ -74,4 +74,12 @@ type FilesystemSharer interface { // UnshareRootFilesystem stops sharing a container bundle // rootfs. UnshareRootFilesystem(context.Context, *Container) error + + // startFileEventWatcher is the event loop to detect changes in + // specific volumes - configmap, secrets, downward-api, projected-volumes + // and copy the changes to the guest + StartFileEventWatcher(context.Context) error + + // Stops the event loop for file watcher + StopFileEventWatcher(context.Context) } diff --git a/src/runtime/virtcontainers/fs_share_darwin.go b/src/runtime/virtcontainers/fs_share_darwin.go index 126ddfb638..5f4bbec17f 100644 --- a/src/runtime/virtcontainers/fs_share_darwin.go +++ b/src/runtime/virtcontainers/fs_share_darwin.go @@ -56,3 +56,10 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) func (f *FilesystemShare) UnshareRootFilesystem(ctx context.Context, c *Container) error { return nil } + +func (f *FilesystemShare) StartFileEventWatcher(ctx context.Context) error { + return nil +} + +func (f *FilesystemShare) StopFileEventWatcher(ctx context.Context) { +} diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index cb5aff391e..5bafb9e403 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -14,9 +14,11 @@ import ( "io/fs" "os" "path/filepath" + "regexp" "sync" "syscall" + "github.com/fsnotify/fsnotify" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -33,14 +35,29 @@ func unmountNoFollow(path string) error { type FilesystemShare struct { sandbox *Sandbox + watcher *fsnotify.Watcher + // The same volume mount can be shared by multiple containers in the same sandbox (pod) + srcDstMap map[string][]string + srcDstMapLock sync.Mutex + eventLoopStarted bool + eventLoopStartedLock sync.Mutex + watcherDoneChannel chan bool sync.Mutex prepared bool } func NewFilesystemShare(s *Sandbox) (FilesystemSharer, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, fmt.Errorf("Creating watcher returned error %w", err) + } + return &FilesystemShare{ - prepared: false, - sandbox: s, + prepared: false, + sandbox: s, + watcherDoneChannel: make(chan bool), + srcDstMap: make(map[string][]string), + watcher: watcher, }, nil } @@ -267,8 +284,59 @@ func (f *FilesystemShare) ShareFile(ctx context.Context, c *Container, m *Mount) } dstPath := filepath.Join(guestPath, srcPath[len(srcRoot):]) + f.Logger().Infof("ShareFile: Copying file from src (%s) to dest (%s)", srcPath, dstPath) + //TODO: Improve the agent protocol, to handle the case for existing symlink. + // Currently for an existing symlink, this will fail with EEXIST. + err = f.sandbox.agent.copyFile(ctx, srcPath, dstPath) + if err != nil { + f.Logger().WithError(err).Error("Failed to copy file") + return err + } - return f.sandbox.agent.copyFile(ctx, srcPath, dstPath) + // Add fsNotify watcher for volume mounts + // Use regex for strict matching instead of strings.Contains + // match for kubernetes.io~configmap, kubernetes.io~secret, kubernetes.io~projected, kubernetes.io~downward-api + // as recommended in review comments for PR #7211 + + // Example directory structure for the volume mounts. + // /var/lib/kubelet/pods/f51ae853-557e-4ce1-b60b-a1101b555612/volumes/kubernetes.io~configmap + // /var/lib/kubelet/pods/f51ae853-557e-4ce1-b60b-a1101b555612/volumes/kubernetes.io~secret + // /var/lib/kubelet/pods/f51ae853-557e-4ce1-b60b-a1101b555612/volumes/kubernetes.io~projected + // /var/lib/kubelet/pods/f51ae853-557e-4ce1-b60b-a1101b555612/volumes/kubernetes.io~downward-api + + // More relaxed regex for the pod UID + // `^/var/lib/kubelet/pods/[a-fA-F0-9\-]+/volumes/kubernetes\.io~(configmap|secret|projected|downward-api)` + + //TODO: Move this to a global variable and compile only once. + regex := regexp.MustCompile(`^/var/lib/kubelet/pods/[a-fA-F0-9\-]{36}/volumes/kubernetes\.io~(configmap|secret|projected|downward-api)`) + if regex.MatchString(srcPath) { + // fsNotify doesn't add watcher recursively. + // So we need to add the watcher for directories under kubernetes.io~configmap, kubernetes.io~secret, + // kubernetes.io~downward-api and kubernetes.io~projected + if info.Mode().IsDir() { + // The cm dir is of the form /var/lib/kubelet/pods//volumes/kubernetes.io~configmap/foo/{..data, key1, key2,...} + // The secret dir is of the form /var/lib/kubelet/pods//volumes/kubernetes.io~secret/foo/{..data, key1, key2,...} + // The projected dir is of the form /var/lib/kubelet/pods//volumes/kubernetes.io~projected/foo/{..data, key1, key2,...} + // The downward-api dir is of the form /var/lib/kubelet/pods//volumes/kubernetes.io~downward-api/foo/{..data, key1, key2,...} + f.Logger().Infof("ShareFile: srcPath(%s) is a directory", srcPath) + err := f.watchDir(srcPath) + if err != nil { + f.Logger().WithError(err).Error("Failed to watch directory") + return err + } + } else { + f.Logger().Infof("ShareFile: srcPath(%s) is not a directory", srcPath) + } + // Add the source and destination to the global map which will be used by the event loop + // to copy the modified content to the destination + f.Logger().Infof("ShareFile: Adding srcPath(%s) dstPath(%s) to srcDstMap", srcPath, dstPath) + // Lock the map before adding the entry + f.srcDstMapLock.Lock() + defer f.srcDstMapLock.Unlock() + f.srcDstMap[srcPath] = append(f.srcDstMap[srcPath], dstPath) + } + + return nil } if err := filepath.WalkDir(srcRoot, walk); err != nil { @@ -278,6 +346,7 @@ func (f *FilesystemShare) ShareFile(ctx context.Context, c *Container, m *Mount) if ignored { return nil, nil } + } else { // These mounts are created in the shared dir mountDest := filepath.Join(getMountPath(f.sandbox.ID()), filename) @@ -512,3 +581,236 @@ func (f *FilesystemShare) UnshareRootFilesystem(ctx context.Context, c *Containe return nil } + +func (f *FilesystemShare) watchDir(source string) error { + + // Add a watcher for the configmap, secret, projected-volumes and downwar-api directories + // /var/lib/kubelet/pods//volumes/{kubernetes.io~configmap, kubernetes.io~secret, kubernetes.io~downward-api, kubernetes.io~projected-volume} + + // Note: From fsNotify docs - https://pkg.go.dev/github.com/fsnotify/fsnotify + // Watching individual files (rather than directories) is generally not + // recommended as many tools update files atomically. Instead of "just" + // writing to the file a temporary file will be written to first, and if + // successful the temporary file is moved to to destination removing the + // original, or some variant thereof. The watcher on the original file is + // now lost, as it no longer exists. + // Instead, watch the parent directory and use Event.Name to filter out files + // you're not interested in. + + // Also fsNotify doesn't add watcher recursively. So we need to walk the root directory and add the required watches + + f.Logger().Infof("watchDir: Add fsnotify watcher for dir (%s)", source) + watchList := f.watcher.WatchList() + + for _, v := range watchList { + if v == source { + f.Logger().Infof("watchDir: Watcher for dir(%s) is already present", source) + return nil + } + } + + err := f.watcher.Add(source) + if err != nil { + f.Logger().WithError(err).Error("watchDir: Failed to add watcher to list") + return err + } + + return nil + +} + +func (f *FilesystemShare) StartFileEventWatcher(ctx context.Context) error { + + // Acquire lock and check if eventLoopStarted + // If not started set the event loop started flag + f.eventLoopStartedLock.Lock() + + // Check if the event loop is already started + if f.eventLoopStarted { + f.Logger().Info("StartFileEventWatcher: Event loop already started, returning") + f.eventLoopStartedLock.Unlock() + return nil + } + + f.Logger().Infof("StartFileEventWatcher: starting the event loop") + + f.eventLoopStarted = true + f.eventLoopStartedLock.Unlock() + + // Regex for the temp directory with timestamp that is used to handle the updates by K8s + // Examples + // /var/lib/kubelet/pods/e33907eb-54c7-4113-a3dc-447f247084cc/volumes/kubernetes.io~secret/foosecret/..2023_07_27_07_13_00.1257228 + // /var/lib/kubelet/pods/e33907eb-54c7-4113-a3dc-447f247084cc/volumes/kubernetes.io~downward-api/fooinfo/..2023_07_27_07_13_00.3704578339 + // The timestamp is of the format 2023_07_27_07_13_00.3704578339 or 2023_07_27_07_13_00.1257228 + + var re = regexp.MustCompile(`(?m)\s*[0-9]{4}_[0-9]{2}_[0-9]{2}_[0-9]{2}_[0-9]{2}_[0-9]{2}.[0-9]*$`) + + f.Logger().Debugf("StartFileEventWatcher: srcDstMap dump %v", f.srcDstMap) + + for { + select { + case event, ok := <-f.watcher.Events: + if !ok { + return fmt.Errorf("StartFileEventWatcher: watcher events channel closed") + } + f.Logger().Infof("StartFileEventWatcher: got an event %s %s", event.Op, event.Name) + if event.Op&fsnotify.Remove == fsnotify.Remove { + // Ref: (kubernetes) pkg/volume/util/atomic_writer.go to understand the configmap/secret update algo + // + // Write does an atomic projection of the given payload into the writer's target + // directory. Input paths must not begin with '..'. + // + // The Write algorithm is: + // + // 1. The payload is validated; if the payload is invalid, the function returns + // 2.  The current timestamped directory is detected by reading the data directory + // symlink + // 3. The old version of the volume is walked to determine whether any + // portion of the payload was deleted and is still present on disk. + // 4. The data in the current timestamped directory is compared to the projected + // data to determine if an update is required. + // 5.  A new timestamped dir is created + // 6. The payload is written to the new timestamped directory + // 7.  Symlinks and directory for new user-visible files are created (if needed). + // + // For example, consider the files: + // /podName + // /user/labels + // /k8s/annotations + // + // The user visible files are symbolic links into the internal data directory: + // /podName -> ..data/podName + // /usr -> ..data/usr + // /k8s -> ..data/k8s + // + // The data directory itself is a link to a timestamped directory with + // the real data: + // /..data -> ..2016_02_01_15_04_05.12345678/ + // 8.  A symlink to the new timestamped directory ..data_tmp is created that will + // become the new data directory + // 9.  The new data directory symlink is renamed to the data directory; rename is atomic + // 10. Old paths are removed from the user-visible portion of the target directory + // 11.  The previous timestamped directory is removed, if it exists + + // In this code, we are relying on the REMOVE event to initate a copy of the updated data. + // This ensures that the required data is updated and available for copying. + // For REMOVE event, the event.Name (source) will be of the form: + // /var/lib/kubelet/pods//volumes//foo/..2023_02_11_09_21_08.2202253910 + // For example, the event.Name (source) for configmap update will like this: + // /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo/..2023_02_11_09_21_08.2202253910 + + source := event.Name + f.Logger().Infof("StartFileEventWatcher: source for the event: %s", source) + if re.FindString(source) != "" { + // This block will be entered when the timestamped directory is removed. + // This also indicates that foo/..data contains the updated info + + volumeDir := filepath.Dir(source) + f.Logger().Infof("StartFileEventWatcher: volumeDir (%s)", volumeDir) + // eg. volumDir = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo + + dataDir := filepath.Join(volumeDir, "..data") + f.Logger().Infof("StartFileEventWatcher: dataDir (%s)", dataDir) + // eg. dataDir = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo/..data + + // Handle different destination for the same source + // Acquire srcDstMapLock before reading srcDstMap + f.srcDstMapLock.Lock() + for _, destination := range f.srcDstMap[dataDir] { + f.Logger().Infof("StartFileEventWatcher: Copy file from src (%s) to dst (%s)", dataDir, destination) + // We explicitly ignore any errors here. Copy will continue for other files + // Errors are logged in the copyFilesFromDataDir method + _ = f.copyFilesFromDataDir(dataDir, destination) + } + f.srcDstMapLock.Unlock() + } + } + case err, ok := <-f.watcher.Errors: + if !ok { + return fmt.Errorf("StartFileEventWatcher: watcher error channel closed") + } + // We continue explicitly here to avoid exiting the watcher loop + f.Logger().Infof("StartFileEventWatcher: got an error event (%v)", err) + continue + case <-f.watcherDoneChannel: + f.Logger().Info("StartFileEventWatcher: watcher closed") + f.watcher.Close() + return nil + } + } +} + +func (f *FilesystemShare) copyFilesFromDataDir(src, dst string) error { + + // The src is a symlink and is of the following form: + // /var/lib/kubelet/pods//volumes//foo/..data + // eg, for configmap, src = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo/..data + // The dst is of the following form: + // /run/kata-containers/shared/containers/-/..data + // eg. dst = /run/kata-containers/shared/containers/e70739a6cc38daf15de916b4d22aad035d42bc977024f2c8cae6b0b607251d44-39407b03e4b448f1-config-volume/..data + + // Get the symlink target + // eg. srcdir = ..2023_02_09_06_40_51.2326009790 + srcdir, err := os.Readlink(src) + if err != nil { + f.Logger().Infof("copyFilesFromDataDir: Reading data symlink returned error (%v)", err) + return err + } + + // Get the base directory path of src + volumeDir := filepath.Dir(src) + // eg. volumeDir = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo + + dataDir := filepath.Join(volumeDir, srcdir) + // eg. dataDir = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo/..2023_02_09_06_40_51.2326009790 + + f.Logger().Infof("copyFilesFromDataDir: full path to data symlink (%s)", dataDir) + + // Using WalkDir is more efficient than Walk + err = filepath.WalkDir(dataDir, + func(path string, d fs.DirEntry, err error) error { + if err != nil { + f.Logger().Infof("copyFilesFromDataDir: Error in file walk %v", err) + return err + } + + // eg. path = /var/lib/kubelet/pods/b44e3261-7cf0-48d3-83b4-6094bba95dc8/volumes/kubernetes.io~configmap/foo/..2023_02_09_06_40_51.2326009790/{key1, key2, ...} + f.Logger().Infof("copyFilesFromDataDir: path (%s)", path) + if !d.IsDir() { + // Using filePath.Rel to handle these cases + // /var/lib/kubelet/pods/2481b69e-9ac8-475a-9e11-88af1daca60e/volumes/kubernetes.io~projected/all-in-one/..2023_02_13_12_35_49.1380323032/config-dir1/config.file1 + // /var/lib/kubelet/pods/2481b69e-9ac8-475a-9e11-88af1daca60e/volumes/kubernetes.io~projected/all-in-one/..2023_02_13_12_35_49.1380323032/config.file2 + rel, err := filepath.Rel(dataDir, path) + if err != nil { + f.Logger().Infof("copyFilesFromDataDir: Unable to get relative path") + return err + } + f.Logger().Debugf("copyFilesFromDataDir: dataDir(%s), path(%s), rel(%s)", dataDir, path, rel) + // Form the destination path in the guest + dstFile := filepath.Join(dst, rel) + f.Logger().Infof("copyFilesFromDataDir: Copying file %s to dst %s", path, dstFile) + err = f.sandbox.agent.copyFile(context.Background(), path, dstFile) + if err != nil { + f.Logger().Infof("copyFilesFromDataDir: Error in copying file %v", err) + return err + } + f.Logger().Infof("copyFilesFromDataDir: Successfully copied file (%s)", path) + } + return nil + }) + + if err != nil { + f.Logger().Infof("copyFilesFromDataDir: Error in filepath.WalkDir (%v)", err) + return err + } + + f.Logger().Infof("copyFilesFromDataDir: Done") + return nil +} + +func (f *FilesystemShare) StopFileEventWatcher(ctx context.Context) { + + f.Logger().Info("StopFileEventWatcher: Closing watcher") + close(f.watcherDoneChannel) + +} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 3a3c275bb5..aba798a905 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -331,6 +331,7 @@ func (s *Sandbox) Release(ctx context.Context) error { if s.monitor != nil { s.monitor.stop() } + s.fsShare.StopFileEventWatcher(ctx) s.hypervisor.Disconnect(ctx) return s.agent.disconnect(ctx) } @@ -613,6 +614,27 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor return nil, err } + // Start the event loop if not already started when fs sharing is not used + if sandboxConfig.HypervisorConfig.SharedFS == config.NoSharedFS { + // Start the StartFileEventWatcher method as a goroutine + // to monitor the file events. + go func() { + if err := s.fsShare.StartFileEventWatcher(ctx); err != nil { + s.Logger().WithError(err).Error("Failed to start file event watcher") + return + } + }() + + // Stop the file event watcher on error + defer func() { + if retErr != nil { + s.Logger().WithError(retErr).Error("Stopping File Event Watcher") + s.fsShare.StopFileEventWatcher(ctx) + } + }() + + } + coldPlugVFIO, err := s.coldOrHotPlugVFIO(&sandboxConfig) if err != nil { return nil, err From afe1a6ac5ae6f5e1d15c6a3b4c829594706f2c01 Mon Sep 17 00:00:00 2001 From: Yohei Ueda Date: Fri, 17 Jun 2022 23:37:13 +0900 Subject: [PATCH 3/4] agent: support copying of directories and symlinks This patch allows copying of directories and symlinks when static file copying is used between host and guest. This change is necessary to support recursive file copying between shim and agent. Signed-off-by: Yohei Ueda (cherry picked from commit de232b8030eba9908557ac6546b97e8044685bfc) --- src/agent/src/rpc.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 30d14d559d..b162d23bfe 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -8,8 +8,9 @@ use rustjail::{pipestream::PipeStream, process::StreamType}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ReadHalf}; use tokio::sync::Mutex; -use std::ffi::CString; +use std::ffi::{CString, OsStr}; use std::io; +use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::sync::Arc; use ttrpc::{ @@ -1856,6 +1857,38 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { } } + let sflag = stat::SFlag::from_bits_truncate(req.file_mode); + + if sflag.contains(stat::SFlag::S_IFDIR) { + fs::create_dir(&path).or_else(|e| { + if e.kind() != std::io::ErrorKind::AlreadyExists { + return Err(e); + } + Ok(()) + })?; + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(req.file_mode))?; + + unistd::chown( + &path, + Some(Uid::from_raw(req.uid as u32)), + Some(Gid::from_raw(req.gid as u32)), + )?; + + return Ok(()); + } + + if sflag.contains(stat::SFlag::S_IFLNK) { + let src = PathBuf::from(OsStr::from_bytes(&req.data)); + unistd::symlinkat(&src, None, &path)?; + let path_str = CString::new(path.as_os_str().as_bytes())?; + + let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) }; + Errno::result(ret).map(drop)?; + + return Ok(()); + } + let mut tmpfile = path.clone(); tmpfile.set_extension("tmp"); From 8815ed0665bb7bcaee7c70bc2686f319fcdaf514 Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 17 Jul 2023 17:17:56 +0100 Subject: [PATCH 4/4] runtime: Remove config warnings Remove configuration file shared_fs = none warnings now that there is a solution to updating configMaps, secrets etc Fixes: #7210 Signed-off-by: stevenhorsman --- src/runtime/config/configuration-clh.toml.in | 4 ---- src/runtime/config/configuration-qemu-nvidia-gpu.toml.in | 4 ---- src/runtime/config/configuration-qemu-sev.toml.in | 4 ---- src/runtime/config/configuration-qemu-snp.toml.in | 4 ---- src/runtime/config/configuration-qemu-tdx.toml.in | 4 ---- src/runtime/config/configuration-qemu.toml.in | 4 ---- 6 files changed, 24 deletions(-) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index fd906ff0a0..b9eb0d8d88 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -132,10 +132,6 @@ default_maxmemory = @DEFMAXMEMSZ@ # - virtio-fs (default) # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index 4861cb1ed6..79adfff8a8 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -179,10 +179,6 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-9p # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in index 413ae1950e..56da05d203 100644 --- a/src/runtime/config/configuration-qemu-sev.toml.in +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -165,10 +165,6 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-9p # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_SEV_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index 54551cba52..290624e89f 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -177,10 +177,6 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-9p # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_SNP_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index fcf1dbe3d2..ab0b57a21e 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -173,10 +173,6 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-9p # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_TDX_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index fc38a0866e..9ca517d7c4 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -179,10 +179,6 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-9p # - virtio-fs-nydus # - none -# WARNING: "none" should be carefully used, and only used in very few specific cases, as -# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing -# issues with rotation of secrets, certs, or configurations via kubernetes objects like -# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon.