From 563c91a2fd59e6ad212924ba6b32760b993ab85f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 19:50:20 +0000 Subject: [PATCH] fix(deps): update module github.com/containers/storage to v1.44.0 Signed-off-by: Renovate Bot --- go.mod | 10 +- go.sum | 15 +- vendor/github.com/BurntSushi/toml/decode.go | 42 +- .../BurntSushi/toml/decode_go116.go | 4 +- vendor/github.com/BurntSushi/toml/doc.go | 22 +- vendor/github.com/BurntSushi/toml/encode.go | 40 +- vendor/github.com/BurntSushi/toml/error.go | 67 +- vendor/github.com/BurntSushi/toml/lex.go | 2 +- vendor/github.com/BurntSushi/toml/meta.go | 4 +- .../Microsoft/hcsshim/internal/hcs/errors.go | 6 +- .../github.com/containers/storage/.cirrus.yml | 21 +- vendor/github.com/containers/storage/Makefile | 4 +- vendor/github.com/containers/storage/VERSION | 2 +- .../containers/storage/containers.go | 303 ++- .../containers/storage/deprecated.go | 216 ++ .../containers/storage/drivers/aufs/aufs.go | 2 +- .../containers/storage/drivers/aufs/mount.go | 1 + .../containers/storage/drivers/btrfs/btrfs.go | 2 +- .../drivers/btrfs/dummy_unsupported.go | 1 + .../storage/drivers/btrfs/version.go | 1 + .../containers/storage/drivers/chown.go | 6 +- .../storage/drivers/chown_darwin.go | 6 +- .../storage/drivers/copy/copy_unsupported.go | 7 +- .../drivers/devmapper/devmapper_doc.go | 1 + .../storage/drivers/devmapper/driver.go | 2 +- .../storage/drivers/devmapper/mount.go | 1 + .../containers/storage/drivers/driver.go | 37 +- .../storage/drivers/driver_solaris.go | 1 + .../storage/drivers/driver_unsupported.go | 1 + .../containers/storage/drivers/fsdiff.go | 20 +- .../storage/drivers/overlay/idmapped_utils.go | 10 +- .../storage/drivers/overlay/mount.go | 97 +- .../storage/drivers/overlay/overlay.go | 62 +- .../storage/drivers/overlay/overlay_cgo.go | 1 + .../storage/drivers/overlay/overlay_nocgo.go | 1 + .../drivers/overlay/overlay_unsupported.go | 1 + .../storage/drivers/quota/projectquota.go | 56 +- .../storage/drivers/register/register_aufs.go | 1 + .../drivers/register/register_btrfs.go | 1 + .../drivers/register/register_devicemapper.go | 1 + .../drivers/register/register_overlay.go | 1 + .../storage/drivers/register/register_zfs.go | 1 + .../containers/storage/drivers/template.go | 3 +- .../storage/drivers/vfs/copy_unsupported.go | 1 + .../containers/storage/drivers/vfs/driver.go | 6 +- .../storage/drivers/windows/windows.go | 2 +- .../containers/storage/drivers/zfs/zfs.go | 2 +- .../storage/drivers/zfs/zfs_unsupported.go | 9 +- .../github.com/containers/storage/images.go | 332 +-- .../github.com/containers/storage/layers.go | 507 ++-- .../containers/storage/pkg/archive/archive.go | 27 +- .../storage/pkg/archive/archive_110.go | 1 + .../storage/pkg/archive/archive_19.go | 1 + .../storage/pkg/archive/archive_bsd.go | 19 + .../storage/pkg/archive/archive_freebsd.go | 129 - .../storage/pkg/archive/archive_linux.go | 19 + .../storage/pkg/archive/archive_other.go | 1 + .../storage/pkg/archive/archive_unix.go | 27 +- .../containers/storage/pkg/archive/changes.go | 2 +- .../storage/pkg/archive/changes_unix.go | 2 + .../storage/pkg/archive/copy_unix.go | 1 + .../containers/storage/pkg/archive/diff.go | 21 +- .../storage/pkg/archive/fflags_bsd.go | 167 ++ .../storage/pkg/archive/fflags_unsupported.go | 21 + .../storage/pkg/archive/time_unsupported.go | 1 + .../storage/pkg/chrootarchive/archive.go | 2 +- .../pkg/chrootarchive/archive_darwin.go | 6 +- .../pkg/chrootarchive/archive_windows.go | 2 +- .../storage/pkg/chrootarchive/chroot_unix.go | 1 + .../storage/pkg/chrootarchive/diff_darwin.go | 4 +- .../storage/pkg/chrootarchive/jsoniter.go | 3 + .../storage/pkg/chunked/compression.go | 252 -- .../storage/pkg/chunked/compression_linux.go | 259 ++ .../pkg/chunked/compressor/compressor.go | 12 +- .../pkg/chunked/internal/compression.go | 8 +- .../containers/storage/pkg/chunked/storage.go | 2 +- .../storage/pkg/devicemapper/devmapper_log.go | 1 + .../pkg/devicemapper/devmapper_wrapper.go | 1 + .../devmapper_wrapper_deferred_remove.go | 1 + .../devicemapper/devmapper_wrapper_dynamic.go | 1 + .../devmapper_wrapper_no_deferred_remove.go | 1 + .../devicemapper/devmapper_wrapper_static.go | 1 + .../storage/pkg/devicemapper/ioctl.go | 1 + .../storage/pkg/dmesg/dmesg_linux.go | 1 + .../storage/pkg/fileutils/fileutils.go | 6 +- .../storage/pkg/homedir/homedir_others.go | 1 + .../storage/pkg/homedir/homedir_unix.go | 2 +- .../containers/storage/pkg/idtools/idtools.go | 4 +- .../pkg/idtools/idtools_unsupported.go | 1 + .../storage/pkg/idtools/idtools_windows.go | 1 + .../storage/pkg/ioutils/fswriters.go | 13 +- .../pkg/ioutils/fswriters_unsupported.go | 1 + .../storage/pkg/lockfile/lockfile.go | 13 +- .../storage/pkg/lockfile/lockfile_unix.go | 76 +- .../storage/pkg/lockfile/lockfile_windows.go | 26 +- .../storage/pkg/loopback/attach_loopback.go | 1 + .../containers/storage/pkg/loopback/ioctl.go | 1 + .../storage/pkg/loopback/loop_wrapper.go | 1 + .../storage/pkg/loopback/loopback.go | 1 + .../storage/pkg/mount/unmount_unix.go | 27 +- .../storage/pkg/mount/unmount_unsupported.go | 1 + .../storage/pkg/parsers/kernel/kernel.go | 1 + .../pkg/parsers/kernel/kernel_darwin.go | 2 +- .../storage/pkg/parsers/kernel/kernel_unix.go | 1 + .../pkg/parsers/kernel/kernel_windows.go | 1 + .../pkg/parsers/kernel/uname_unsupported.go | 11 +- .../parsers/kernel/uname_unsupported_type.go | 11 + .../storage/pkg/reexec/command_freebsd.go | 1 + .../storage/pkg/reexec/command_linux.go | 1 + .../storage/pkg/reexec/command_unsupported.go | 1 + .../storage/pkg/reexec/command_windows.go | 1 + .../storage/pkg/system/chtimes_unix.go | 1 + .../storage/pkg/system/chtimes_windows.go | 1 + .../containers/storage/pkg/system/init.go | 2 +- .../storage/pkg/system/lchflags_bsd.go | 56 + .../storage/pkg/system/lcow_unix.go | 1 + .../storage/pkg/system/lstat_unix.go | 3 +- .../storage/pkg/system/mknod_windows.go | 1 + .../storage/pkg/system/path_unix.go | 1 + .../storage/pkg/system/process_unix.go | 1 + .../containers/storage/pkg/system/rm.go | 19 +- .../storage/pkg/system/rm_freebsd.go | 17 +- .../storage/pkg/system/stat_common.go | 13 + .../storage/pkg/system/stat_freebsd.go | 15 +- .../storage/pkg/system/stat_unix.go | 2 + .../storage/pkg/system/stat_windows.go | 1 + .../containers/storage/pkg/system/umask.go | 1 + .../storage/pkg/system/umask_windows.go | 1 + .../storage/pkg/system/utimes_unsupported.go | 1 + .../storage/pkg/system/xattrs_unsupported.go | 1 + .../storage/pkg/truncindex/truncindex.go | 10 +- .../storage/pkg/unshare/getenv_linux_cgo.go | 1 + .../storage/pkg/unshare/getenv_linux_nocgo.go | 1 + .../containers/storage/pkg/unshare/unshare.go | 17 - .../storage/pkg/unshare/unshare_darwin.go | 1 + .../storage/pkg/unshare/unshare_gccgo.go | 1 + .../storage/pkg/unshare/unshare_linux.go | 19 +- vendor/github.com/containers/storage/store.go | 2265 +++++++---------- .../containers/storage/types/options.go | 31 +- .../storage/types/options_darwin.go | 6 +- .../storage/types/options_freebsd.go | 5 +- .../containers/storage/types/options_linux.go | 5 +- .../storage/types/options_windows.go | 5 +- .../containers/storage/types/utils.go | 2 +- .../github.com/containers/storage/userns.go | 45 +- .../github.com/klauspost/compress/README.md | 7 + .../klauspost/compress/zstd/blockdec.go | 2 +- .../klauspost/compress/zstd/decoder.go | 2 +- .../klauspost/compress/zstd/enc_fast.go | 2 +- .../klauspost/compress/zstd/framedec.go | 11 +- vendor/modules.txt | 12 +- 151 files changed, 3061 insertions(+), 2674 deletions(-) create mode 100644 vendor/github.com/containers/storage/deprecated.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/archive_bsd.go delete mode 100644 vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/fflags_bsd.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/fflags_unsupported.go create mode 100644 vendor/github.com/containers/storage/pkg/chunked/compression_linux.go create mode 100644 vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported_type.go create mode 100644 vendor/github.com/containers/storage/pkg/system/lchflags_bsd.go create mode 100644 vendor/github.com/containers/storage/pkg/system/stat_common.go diff --git a/go.mod b/go.mod index 0d655b71..b51dad2c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/containers/common v0.50.1 github.com/containers/image/v5 v5.23.1-0.20221019175208-1dd254487708 github.com/containers/ocicrypt v1.1.6 - github.com/containers/storage v1.43.0 + github.com/containers/storage v1.44.0 github.com/docker/distribution v2.8.1+incompatible github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc2 @@ -22,13 +22,13 @@ require ( ) require ( - github.com/BurntSushi/toml v1.2.0 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/Microsoft/hcsshim v0.9.4 // indirect + github.com/Microsoft/hcsshim v0.9.5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/containerd/cgroups v1.0.4 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -50,7 +50,7 @@ require ( github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.11 // indirect + github.com/klauspost/compress v1.15.12 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index d8a11ab7..43117e89 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,9 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -207,8 +208,9 @@ github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwT github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.5 h1:AbV+VPfTrIVffukazHcpxmz/sRiE6YaMDzHWR9BXZHo= +github.com/Microsoft/hcsshim v0.9.5/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -451,8 +453,9 @@ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFY github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/stargz-snapshotter/estargz v0.12.0 h1:idtwRTLjk2erqiYhPWy2L844By8NRFYEwYHcXhoIWPM= github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M= +github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0= +github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -493,8 +496,9 @@ github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pA github.com/containers/ocicrypt v1.1.5/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= github.com/containers/ocicrypt v1.1.6 h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnxiFaI= github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= -github.com/containers/storage v1.43.0 h1:P+zulGXA3mqe2GnYmZU0xu87Wy1M0PVHM2ucrgmvTdU= github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s= +github.com/containers/storage v1.44.0 h1:xCFhwILjjU+Hg133d07EVCgaaXn0ileGcVsAUcV8tDY= +github.com/containers/storage v1.44.0/go.mod h1:HSfx7vUXwKPatPMqhgMw3mI3c3ijIJPZV5O0sj/mVxI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -1167,8 +1171,9 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= +github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0= diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index 09523315..0ca1dc4f 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -21,7 +21,9 @@ type Unmarshaler interface { UnmarshalTOML(interface{}) error } -// Unmarshal decodes the contents of `data` in TOML format into a pointer `v`. +// Unmarshal decodes the contents of data in TOML format into a pointer v. +// +// See [Decoder] for a description of the decoding process. func Unmarshal(data []byte, v interface{}) error { _, err := NewDecoder(bytes.NewReader(data)).Decode(v) return err @@ -29,13 +31,12 @@ func Unmarshal(data []byte, v interface{}) error { // Decode the TOML data in to the pointer v. // -// See the documentation on Decoder for a description of the decoding process. +// See [Decoder] for a description of the decoding process. func Decode(data string, v interface{}) (MetaData, error) { return NewDecoder(strings.NewReader(data)).Decode(v) } -// DecodeFile is just like Decode, except it will automatically read the -// contents of the file at path and decode it for you. +// DecodeFile reads the contents of a file and decodes it with [Decode]. func DecodeFile(path string, v interface{}) (MetaData, error) { fp, err := os.Open(path) if err != nil { @@ -48,7 +49,7 @@ func DecodeFile(path string, v interface{}) (MetaData, error) { // Primitive is a TOML value that hasn't been decoded into a Go value. // // This type can be used for any value, which will cause decoding to be delayed. -// You can use the PrimitiveDecode() function to "manually" decode these values. +// You can use [PrimitiveDecode] to "manually" decode these values. // // NOTE: The underlying representation of a `Primitive` value is subject to // change. Do not rely on it. @@ -70,15 +71,15 @@ const ( // Decoder decodes TOML data. // -// TOML tables correspond to Go structs or maps (dealer's choice – they can be -// used interchangeably). +// TOML tables correspond to Go structs or maps; they can be used +// interchangeably, but structs offer better type safety. // // TOML table arrays correspond to either a slice of structs or a slice of maps. // -// TOML datetimes correspond to Go time.Time values. Local datetimes are parsed -// in the local timezone. +// TOML datetimes correspond to [time.Time]. Local datetimes are parsed in the +// local timezone. // -// time.Duration types are treated as nanoseconds if the TOML value is an +// [time.Duration] types are treated as nanoseconds if the TOML value is an // integer, or they're parsed with time.ParseDuration() if they're strings. // // All other TOML types (float, string, int, bool and array) correspond to the @@ -90,7 +91,7 @@ const ( // UnmarshalText method. See the Unmarshaler example for a demonstration with // email addresses. // -// Key mapping +// ### Key mapping // // TOML keys can map to either keys in a Go map or field names in a Go struct. // The special `toml` struct tag can be used to map TOML keys to struct fields @@ -168,17 +169,16 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) { return md, md.unify(p.mapping, rv) } -// PrimitiveDecode is just like the other `Decode*` functions, except it -// decodes a TOML value that has already been parsed. Valid primitive values -// can *only* be obtained from values filled by the decoder functions, -// including this method. (i.e., `v` may contain more `Primitive` -// values.) +// PrimitiveDecode is just like the other Decode* functions, except it decodes a +// TOML value that has already been parsed. Valid primitive values can *only* be +// obtained from values filled by the decoder functions, including this method. +// (i.e., v may contain more [Primitive] values.) // -// Meta data for primitive values is included in the meta data returned by -// the `Decode*` functions with one exception: keys returned by the Undecoded -// method will only reflect keys that were decoded. Namely, any keys hidden -// behind a Primitive will be considered undecoded. Executing this method will -// update the undecoded keys in the meta data. (See the example.) +// Meta data for primitive values is included in the meta data returned by the +// Decode* functions with one exception: keys returned by the Undecoded method +// will only reflect keys that were decoded. Namely, any keys hidden behind a +// Primitive will be considered undecoded. Executing this method will update the +// undecoded keys in the meta data. (See the example.) func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { md.context = primValue.context defer func() { md.context = nil }() diff --git a/vendor/github.com/BurntSushi/toml/decode_go116.go b/vendor/github.com/BurntSushi/toml/decode_go116.go index eddfb641..086d0b68 100644 --- a/vendor/github.com/BurntSushi/toml/decode_go116.go +++ b/vendor/github.com/BurntSushi/toml/decode_go116.go @@ -7,8 +7,8 @@ import ( "io/fs" ) -// DecodeFS is just like Decode, except it will automatically read the contents -// of the file at `path` from a fs.FS instance. +// DecodeFS reads the contents of a file from [fs.FS] and decodes it with +// [Decode]. func DecodeFS(fsys fs.FS, path string, v interface{}) (MetaData, error) { fp, err := fsys.Open(path) if err != nil { diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go index 099c4a77..81a7c0fe 100644 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -1,13 +1,11 @@ -/* -Package toml implements decoding and encoding of TOML files. - -This package supports TOML v1.0.0, as listed on https://toml.io - -There is also support for delaying decoding with the Primitive type, and -querying the set of keys in a TOML document with the MetaData type. - -The github.com/BurntSushi/toml/cmd/tomlv package implements a TOML validator, -and can be used to verify if TOML document is valid. It can also be used to -print the type of each key. -*/ +// Package toml implements decoding and encoding of TOML files. +// +// This package supports TOML v1.0.0, as specified at https://toml.io +// +// There is also support for delaying decoding with the Primitive type, and +// querying the set of keys in a TOML document with the MetaData type. +// +// The github.com/BurntSushi/toml/cmd/tomlv package implements a TOML validator, +// and can be used to verify if TOML document is valid. It can also be used to +// print the type of each key. package toml diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index dc8568d1..930e1d52 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -79,12 +79,12 @@ type Marshaler interface { // Encoder encodes a Go to a TOML document. // // The mapping between Go values and TOML values should be precisely the same as -// for the Decode* functions. +// for [Decode]. // // time.Time is encoded as a RFC 3339 string, and time.Duration as its string // representation. // -// The toml.Marshaler and encoder.TextMarshaler interfaces are supported to +// The [Marshaler] and [encoding.TextMarshaler] interfaces are supported to // encoding the value as custom TOML. // // If you want to write arbitrary binary data then you will need to use @@ -130,7 +130,7 @@ func NewEncoder(w io.Writer) *Encoder { } } -// Encode writes a TOML representation of the Go value to the Encoder's writer. +// Encode writes a TOML representation of the Go value to the [Encoder]'s writer. // // An error is returned if the value given cannot be encoded to a valid TOML // document. @@ -261,7 +261,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { enc.eElement(reflect.ValueOf(v)) return } - encPanic(errors.New(fmt.Sprintf("Unable to convert \"%s\" to neither int64 nor float64", n))) + encPanic(fmt.Errorf("unable to convert %q to int64 or float64", n)) } switch rv.Kind() { @@ -504,7 +504,8 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) { if opts.name != "" { keyName = opts.name } - if opts.omitempty && isEmpty(fieldVal) { + + if opts.omitempty && enc.isEmpty(fieldVal) { continue } if opts.omitzero && isZero(fieldVal) { @@ -648,12 +649,26 @@ func isZero(rv reflect.Value) bool { return false } -func isEmpty(rv reflect.Value) bool { +func (enc *Encoder) isEmpty(rv reflect.Value) bool { switch rv.Kind() { case reflect.Array, reflect.Slice, reflect.Map, reflect.String: return rv.Len() == 0 case reflect.Struct: - return reflect.Zero(rv.Type()).Interface() == rv.Interface() + if rv.Type().Comparable() { + return reflect.Zero(rv.Type()).Interface() == rv.Interface() + } + // Need to also check if all the fields are empty, otherwise something + // like this with uncomparable types will always return true: + // + // type a struct{ field b } + // type b struct{ s []string } + // s := a{field: b{s: []string{"AAA"}}} + for i := 0; i < rv.NumField(); i++ { + if !enc.isEmpty(rv.Field(i)) { + return false + } + } + return true case reflect.Bool: return !rv.Bool() } @@ -668,16 +683,15 @@ func (enc *Encoder) newline() { // Write a key/value pair: // -// key = +// key = // // This is also used for "k = v" in inline tables; so something like this will // be written in three calls: // -// ┌────────────────────┐ -// │ ┌───┐ ┌─────┐│ -// v v v v vv -// key = {k = v, k2 = v2} -// +// ┌───────────────────┐ +// │ ┌───┐ ┌────┐│ +// v v v v vv +// key = {k = 1, k2 = 2} func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) { if len(key) == 0 { encPanic(errNoKey) diff --git a/vendor/github.com/BurntSushi/toml/error.go b/vendor/github.com/BurntSushi/toml/error.go index 2ac24e77..f4f390e6 100644 --- a/vendor/github.com/BurntSushi/toml/error.go +++ b/vendor/github.com/BurntSushi/toml/error.go @@ -5,57 +5,60 @@ import ( "strings" ) -// ParseError is returned when there is an error parsing the TOML syntax. -// -// For example invalid syntax, duplicate keys, etc. +// ParseError is returned when there is an error parsing the TOML syntax such as +// invalid syntax, duplicate keys, etc. // // In addition to the error message itself, you can also print detailed location -// information with context by using ErrorWithPosition(): +// information with context by using [ErrorWithPosition]: // -// toml: error: Key 'fruit' was already created and cannot be used as an array. +// toml: error: Key 'fruit' was already created and cannot be used as an array. // -// At line 4, column 2-7: +// At line 4, column 2-7: // -// 2 | fruit = [] -// 3 | -// 4 | [[fruit]] # Not allowed -// ^^^^^ +// 2 | fruit = [] +// 3 | +// 4 | [[fruit]] # Not allowed +// ^^^^^ // -// Furthermore, the ErrorWithUsage() can be used to print the above with some -// more detailed usage guidance: +// [ErrorWithUsage] can be used to print the above with some more detailed usage +// guidance: // -// toml: error: newlines not allowed within inline tables +// toml: error: newlines not allowed within inline tables // -// At line 1, column 18: +// At line 1, column 18: // -// 1 | x = [{ key = 42 # -// ^ +// 1 | x = [{ key = 42 # +// ^ // -// Error help: +// Error help: // -// Inline tables must always be on a single line: +// Inline tables must always be on a single line: // -// table = {key = 42, second = 43} +// table = {key = 42, second = 43} // -// It is invalid to split them over multiple lines like so: +// It is invalid to split them over multiple lines like so: // -// # INVALID -// table = { -// key = 42, -// second = 43 -// } +// # INVALID +// table = { +// key = 42, +// second = 43 +// } // -// Use regular for this: +// Use regular for this: // -// [table] -// key = 42 -// second = 43 +// [table] +// key = 42 +// second = 43 type ParseError struct { Message string // Short technical message. Usage string // Longer message with usage guidance; may be blank. Position Position // Position of the error LastKey string // Last parsed key, may be blank. - Line int // Line the error occurred. Deprecated: use Position. + + // Line the error occurred. + // + // Deprecated: use [Position]. + Line int err error input string @@ -83,7 +86,7 @@ func (pe ParseError) Error() string { // ErrorWithUsage() returns the error with detailed location context. // -// See the documentation on ParseError. +// See the documentation on [ParseError]. func (pe ParseError) ErrorWithPosition() string { if pe.input == "" { // Should never happen, but just in case. return pe.Error() @@ -124,7 +127,7 @@ func (pe ParseError) ErrorWithPosition() string { // ErrorWithUsage() returns the error with detailed location context and usage // guidance. // -// See the documentation on ParseError. +// See the documentation on [ParseError]. func (pe ParseError) ErrorWithUsage() string { m := pe.ErrorWithPosition() if u, ok := pe.err.(interface{ Usage() string }); ok && u.Usage() != "" { diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 28ed4dd3..d4d70871 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -771,7 +771,7 @@ func lexRawString(lx *lexer) stateFn { } // lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'''" has already been consumed and +// a string. It assumes that the beginning ''' has already been consumed and // ignored. func lexMultilineRawString(lx *lexer) stateFn { r := lx.next() diff --git a/vendor/github.com/BurntSushi/toml/meta.go b/vendor/github.com/BurntSushi/toml/meta.go index d284f2a0..71847a04 100644 --- a/vendor/github.com/BurntSushi/toml/meta.go +++ b/vendor/github.com/BurntSushi/toml/meta.go @@ -71,7 +71,7 @@ func (md *MetaData) Keys() []Key { // Undecoded returns all keys that have not been decoded in the order in which // they appear in the original TOML document. // -// This includes keys that haven't been decoded because of a Primitive value. +// This includes keys that haven't been decoded because of a [Primitive] value. // Once the Primitive value is decoded, the keys will be considered decoded. // // Also note that decoding into an empty interface will result in no decoding, @@ -89,7 +89,7 @@ func (md *MetaData) Undecoded() []Key { return undecoded } -// Key represents any TOML key, including key groups. Use (MetaData).Keys to get +// Key represents any TOML key, including key groups. Use [MetaData.Keys] to get // values of this type. type Key []string diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go index e21354ff..295d4b84 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go @@ -154,7 +154,7 @@ func (e *HcsError) Error() string { func (e *HcsError) Temporary() bool { err, ok := e.Err.(net.Error) - return ok && err.Temporary() + return ok && err.Temporary() //nolint:staticcheck } func (e *HcsError) Timeout() bool { @@ -193,7 +193,7 @@ func (e *SystemError) Error() string { func (e *SystemError) Temporary() bool { err, ok := e.Err.(net.Error) - return ok && err.Temporary() + return ok && err.Temporary() //nolint:staticcheck } func (e *SystemError) Timeout() bool { @@ -224,7 +224,7 @@ func (e *ProcessError) Error() string { func (e *ProcessError) Temporary() bool { err, ok := e.Err.(net.Error) - return ok && err.Temporary() + return ok && err.Temporary() //nolint:staticcheck } func (e *ProcessError) Timeout() bool { diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index e3e2b58f..1a4c0e77 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -18,7 +18,6 @@ env: #### Cache-image names to test with (double-quotes around names are critical) ### FEDORA_NAME: "fedora-36" - PRIOR_FEDORA_NAME: "fedora-35" UBUNTU_NAME: "ubuntu-2204" # GCE project where images live @@ -26,7 +25,6 @@ env: # VM Image built in containers/automation_images IMAGE_SUFFIX: "c5878804328480768" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" #### @@ -90,15 +88,6 @@ fedora_testing_task: &fedora_testing journal_log_script: '${_JOURNALCMD} || true' -prior_fedora_testing_task: - <<: *fedora_testing - alias: prior_fedora_testing - name: *std_test_name - env: - OS_NAME: "${PRIOR_FEDORA_NAME}" - VM_IMAGE: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - - ubuntu_testing_task: &ubuntu_testing <<: *fedora_testing alias: ubuntu_testing @@ -117,7 +106,7 @@ lint_task: env: CIRRUS_WORKING_DIR: "/go/src/github.com/containers/storage" container: - image: golang:1.16 + image: golang:1.17 modules_cache: fingerprint_script: cat go.sum folder: $GOPATH/pkg/mod @@ -125,7 +114,7 @@ lint_task: echo "deb http://deb.debian.org/debian stretch-backports main" > /etc/apt/sources.list.d/backports.list apt-get update apt-get install -y libbtrfs-dev libdevmapper-dev - test_script: make lint + test_script: make local-validate && make lint # Update metadata on VM images referenced by this repository state @@ -140,7 +129,6 @@ meta_task: # Space-separated list of images used by this repository state IMGNAMES: |- ${FEDORA_CACHE_IMAGE_NAME} - ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" @@ -154,7 +142,7 @@ meta_task: vendor_task: container: - image: golang:1.16 + image: golang:1.17 modules_cache: fingerprint_script: cat go.sum folder: $GOPATH/pkg/mod @@ -167,11 +155,10 @@ success_task: depends_on: - lint - fedora_testing - - prior_fedora_testing - ubuntu_testing - meta - vendor container: - image: golang:1.16 + image: golang:1.17 clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"' # Source code not needed script: /bin/true diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 244576d5..28c7a2dd 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -92,7 +92,7 @@ local-test-integration: local-binary ## run the integration tests on the host (r test-integration: local-binary ## run the integration tests using VMs $(RUNINVM) $(MAKE) local-$@ -local-validate: ## validate DCO and gofmt on the host +local-validate: install.tools ## validate DCO and gofmt on the host @./hack/git-validation.sh @./hack/gofmt.sh @@ -120,6 +120,6 @@ vendor-in-container: podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src golang make vendor vendor: - $(GO) mod tidy + $(GO) mod tidy -compat=1.17 $(GO) mod vendor $(GO) mod verify diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index b978278f..372cf402 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.43.0 +1.44.0 diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index 4f2b61f5..d3223cd4 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -66,12 +66,25 @@ type Container struct { Flags map[string]interface{} `json:"flags,omitempty"` } -// ContainerStore provides bookkeeping for information about Containers. -type ContainerStore interface { - FileBasedStore - MetadataStore - ContainerBigDataStore - FlaggableStore +// rwContainerStore provides bookkeeping for information about Containers. +type rwContainerStore interface { + metadataStore + containerBigDataStore + flaggableStore + + // startWriting makes sure the store is fresh, and locks it for writing. + // If this succeeds, the caller MUST call stopWriting(). + startWriting() error + + // stopWriting releases locks obtained by startWriting. + stopWriting() + + // startReading makes sure the store is fresh, and locks it for reading. + // If this succeeds, the caller MUST call stopReading(). + startReading() error + + // stopReading releases locks obtained by startReading. + stopReading() // Create creates a container that has a specified ID (or generates a // random one if an empty value is supplied) and optional names, @@ -81,18 +94,8 @@ type ContainerStore interface { // convenience of the caller, nothing more. Create(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) - // SetNames updates the list of names associated with the container - // with the specified ID. - // Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. - SetNames(id string, names []string) error - - // AddNames adds the supplied values to the list of names associated with the container with - // the specified id. - AddNames(id string, names []string) error - - // RemoveNames removes the supplied values from the list of names associated with the container with - // the specified id. - RemoveNames(id string, names []string) error + // updateNames modifies names associated with a container based on (op, names). + updateNames(id string, names []string, op updateNameOperation) error // Get retrieves information about a container given an ID or name. Get(id string) (*Container, error) @@ -157,12 +160,12 @@ func (c *Container) ProcessLabel() string { } func (c *Container) MountOpts() []string { - switch c.Flags[mountOptsFlag].(type) { + switch value := c.Flags[mountOptsFlag].(type) { case []string: - return c.Flags[mountOptsFlag].([]string) + return value case []interface{}: var mountOpts []string - for _, v := range c.Flags[mountOptsFlag].([]interface{}) { + for _, v := range value { if flag, ok := v.(string); ok { mountOpts = append(mountOpts, flag) } @@ -173,6 +176,80 @@ func (c *Container) MountOpts() []string { } } +// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +// +// This is an internal implementation detail of containerStore construction, every other caller +// should use startWriting() instead. +func (r *containerStore) startWritingWithReload(canReload bool) error { + r.lockfile.Lock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if canReload { + if err := r.reloadIfChanged(true); err != nil { + return err + } + } + + succeeded = true + return nil +} + +// startWriting makes sure the store is fresh, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +func (r *containerStore) startWriting() error { + return r.startWritingWithReload(true) +} + +// stopWriting releases locks obtained by startWriting. +func (r *containerStore) stopWriting() { + r.lockfile.Unlock() +} + +// startReading makes sure the store is fresh, and locks it for reading. +// If this succeeds, the caller MUST call stopReading(). +func (r *containerStore) startReading() error { + r.lockfile.RLock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if err := r.reloadIfChanged(false); err != nil { + return err + } + + succeeded = true + return nil +} + +// stopReading releases locks obtained by startReading. +func (r *containerStore) stopReading() { + r.lockfile.Unlock() +} + +// reloadIfChanged reloads the contents of the store from disk if it is changed. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *containerStore) reloadIfChanged(lockedForWriting bool) error { + r.loadMut.Lock() + defer r.loadMut.Unlock() + + modified, err := r.lockfile.Modified() + if err == nil && modified { + return r.load(lockedForWriting) + } + return err +} + func (r *containerStore) Containers() ([]Container, error) { containers := make([]Container, len(r.containers)) for i := range r.containers { @@ -193,48 +270,60 @@ func (r *containerStore) datapath(id, key string) string { return filepath.Join(r.datadir(id), makeBigDataBaseName(key)) } -func (r *containerStore) Load() error { +// load reloads the contents of the store from disk. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *containerStore) load(lockedForWriting bool) error { needSave := false rpath := r.containerspath() data, err := os.ReadFile(rpath) if err != nil && !os.IsNotExist(err) { return err } + containers := []*Container{} - layers := make(map[string]*Container) - idlist := []string{} - ids := make(map[string]*Container) - names := make(map[string]*Container) - if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil { - idlist = make([]string, 0, len(containers)) - for n, container := range containers { - idlist = append(idlist, container.ID) - ids[container.ID] = containers[n] - layers[container.LayerID] = containers[n] - for _, name := range container.Names { - if conflict, ok := names[name]; ok { - r.removeName(conflict, name) - needSave = true - } - names[name] = containers[n] - } + if len(data) != 0 { + if err := json.Unmarshal(data, &containers); err != nil { + return fmt.Errorf("loading %q: %w", rpath, err) } } + idlist := make([]string, 0, len(containers)) + layers := make(map[string]*Container) + ids := make(map[string]*Container) + names := make(map[string]*Container) + for n, container := range containers { + idlist = append(idlist, container.ID) + ids[container.ID] = containers[n] + layers[container.LayerID] = containers[n] + for _, name := range container.Names { + if conflict, ok := names[name]; ok { + r.removeName(conflict, name) + needSave = true + } + names[name] = containers[n] + } + } + r.containers = containers - r.idindex = truncindex.NewTruncIndex(idlist) + r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store. r.byid = ids r.bylayer = layers r.byname = names if needSave { + if !lockedForWriting { + // Eventually, the callers should be modified to retry with a write lock, instead. + return errors.New("container store is inconsistent and the current caller does not hold a write lock") + } return r.Save() } return nil } +// Save saves the contents of the store to disk. It should be called with +// the lock held, locked for writing. func (r *containerStore) Save() error { - if !r.Locked() { - return errors.New("container store is not locked") - } + r.lockfile.AssertLockedForWriting() rpath := r.containerspath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -243,11 +332,13 @@ func (r *containerStore) Save() error { if err != nil { return err } - defer r.Touch() - return ioutils.AtomicWriteFile(rpath, jdata, 0600) + if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil { + return err + } + return r.lockfile.Touch() } -func newContainerStore(dir string) (ContainerStore, error) { +func newContainerStore(dir string) (rwContainerStore, error) { if err := os.MkdirAll(dir, 0700); err != nil { return nil, err } @@ -255,8 +346,6 @@ func newContainerStore(dir string) (ContainerStore, error) { if err != nil { return nil, err } - lockfile.Lock() - defer lockfile.Unlock() cstore := containerStore{ lockfile: lockfile, dir: dir, @@ -265,7 +354,11 @@ func newContainerStore(dir string) (ContainerStore, error) { bylayer: make(map[string]*Container), byname: make(map[string]*Container), } - if err := cstore.Load(); err != nil { + if err := cstore.startWritingWithReload(false); err != nil { + return nil, err + } + defer cstore.stopWriting() + if err := cstore.load(true); err != nil { return nil, err } return &cstore, nil @@ -337,31 +430,31 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat if err := hasOverlappingRanges(options.GIDMap); err != nil { return nil, err } - if err == nil { - container = &Container{ - ID: id, - Names: names, - ImageID: image, - LayerID: layer, - Metadata: metadata, - BigDataNames: []string{}, - BigDataSizes: make(map[string]int64), - BigDataDigests: make(map[string]digest.Digest), - Created: time.Now().UTC(), - Flags: copyStringInterfaceMap(options.Flags), - UIDMap: copyIDMap(options.UIDMap), - GIDMap: copyIDMap(options.GIDMap), - } - r.containers = append(r.containers, container) - r.byid[id] = container - r.idindex.Add(id) - r.bylayer[layer] = container - for _, name := range names { - r.byname[name] = container - } - err = r.Save() - container = copyContainer(container) + container = &Container{ + ID: id, + Names: names, + ImageID: image, + LayerID: layer, + Metadata: metadata, + BigDataNames: []string{}, + BigDataSizes: make(map[string]int64), + BigDataDigests: make(map[string]digest.Digest), + Created: time.Now().UTC(), + Flags: copyStringInterfaceMap(options.Flags), + UIDMap: copyIDMap(options.UIDMap), + GIDMap: copyIDMap(options.GIDMap), } + r.containers = append(r.containers, container) + r.byid[id] = container + // This can only fail on duplicate IDs, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // Implementing recovery from an unlikely and unimportant failure here would be too risky. + _ = r.idindex.Add(id) + r.bylayer[layer] = container + for _, name := range names { + r.byname[name] = container + } + err = r.Save() + container = copyContainer(container) return container, err } @@ -384,19 +477,6 @@ func (r *containerStore) removeName(container *Container, name string) { container.Names = stringSliceWithoutValue(container.Names, name) } -// Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. -func (r *containerStore) SetNames(id string, names []string) error { - return r.updateNames(id, names, setNames) -} - -func (r *containerStore) AddNames(id string, names []string) error { - return r.updateNames(id, names, addNames) -} - -func (r *containerStore) RemoveNames(id string, names []string) error { - return r.updateNames(id, names, removeNames) -} - func (r *containerStore) updateNames(id string, names []string, op updateNameOperation) error { container, ok := r.lookup(id) if !ok { @@ -434,7 +514,9 @@ func (r *containerStore) Delete(id string) error { } } delete(r.byid, id) - r.idindex.Delete(id) + // This can only fail if the ID is already missing, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // The store’s Delete method is used on various paths to recover from failures, so this should be robust against partially missing data. + _ = r.idindex.Delete(id) delete(r.bylayer, container.LayerID) for _, name := range container.Names { delete(r.byname, name) @@ -612,50 +694,3 @@ func (r *containerStore) Wipe() error { } return nil } - -func (r *containerStore) Lock() { - r.lockfile.Lock() -} - -func (r *containerStore) RecursiveLock() { - r.lockfile.RecursiveLock() -} - -func (r *containerStore) RLock() { - r.lockfile.RLock() -} - -func (r *containerStore) Unlock() { - r.lockfile.Unlock() -} - -func (r *containerStore) Touch() error { - return r.lockfile.Touch() -} - -func (r *containerStore) Modified() (bool, error) { - return r.lockfile.Modified() -} - -func (r *containerStore) IsReadWrite() bool { - return r.lockfile.IsReadWrite() -} - -func (r *containerStore) TouchedSince(when time.Time) bool { - return r.lockfile.TouchedSince(when) -} - -func (r *containerStore) Locked() bool { - return r.lockfile.Locked() -} - -func (r *containerStore) ReloadIfChanged() error { - r.loadMut.Lock() - defer r.loadMut.Unlock() - - modified, err := r.Modified() - if err == nil && modified { - return r.Load() - } - return err -} diff --git a/vendor/github.com/containers/storage/deprecated.go b/vendor/github.com/containers/storage/deprecated.go new file mode 100644 index 00000000..04972d83 --- /dev/null +++ b/vendor/github.com/containers/storage/deprecated.go @@ -0,0 +1,216 @@ +package storage + +import ( + "io" + "time" + + drivers "github.com/containers/storage/drivers" + "github.com/containers/storage/pkg/archive" + digest "github.com/opencontainers/go-digest" +) + +// The type definitions in this file exist ONLY to maintain formal API compatibility. +// DO NOT ADD ANY NEW METHODS TO THESE INTERFACES. + +// ROFileBasedStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROFileBasedStore interface { + Locker + Load() error + ReloadIfChanged() error +} + +// RWFileBasedStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type RWFileBasedStore interface { + Save() error +} + +// FileBasedStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type FileBasedStore interface { + ROFileBasedStore + RWFileBasedStore +} + +// ROMetadataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROMetadataStore interface { + Metadata(id string) (string, error) +} + +// RWMetadataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type RWMetadataStore interface { + SetMetadata(id, metadata string) error +} + +// MetadataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type MetadataStore interface { + ROMetadataStore + RWMetadataStore +} + +// ROBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROBigDataStore interface { + BigData(id, key string) ([]byte, error) + BigDataSize(id, key string) (int64, error) + BigDataDigest(id, key string) (digest.Digest, error) + BigDataNames(id string) ([]string, error) +} + +// RWImageBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type RWImageBigDataStore interface { + SetBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error +} + +// ContainerBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ContainerBigDataStore interface { + ROBigDataStore + SetBigData(id, key string, data []byte) error +} + +// ROLayerBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROLayerBigDataStore interface { + BigData(id, key string) (io.ReadCloser, error) + BigDataNames(id string) ([]string, error) +} + +// RWLayerBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type RWLayerBigDataStore interface { + SetBigData(id, key string, data io.Reader) error +} + +// LayerBigDataStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type LayerBigDataStore interface { + ROLayerBigDataStore + RWLayerBigDataStore +} + +// FlaggableStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type FlaggableStore interface { + ClearFlag(id string, flag string) error + SetFlag(id string, flag string, value interface{}) error +} + +// ContainerStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ContainerStore interface { + FileBasedStore + MetadataStore + ContainerBigDataStore + FlaggableStore + Create(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) + SetNames(id string, names []string) error + AddNames(id string, names []string) error + RemoveNames(id string, names []string) error + Get(id string) (*Container, error) + Exists(id string) bool + Delete(id string) error + Wipe() error + Lookup(name string) (string, error) + Containers() ([]Container, error) +} + +// ROImageStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROImageStore interface { + ROFileBasedStore + ROMetadataStore + ROBigDataStore + Exists(id string) bool + Get(id string) (*Image, error) + Lookup(name string) (string, error) + Images() ([]Image, error) + ByDigest(d digest.Digest) ([]*Image, error) +} + +// ImageStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ImageStore interface { + ROImageStore + RWFileBasedStore + RWMetadataStore + RWImageBigDataStore + FlaggableStore + Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error) + SetNames(id string, names []string) error + AddNames(id string, names []string) error + RemoveNames(id string, names []string) error + Delete(id string) error + Wipe() error +} + +// ROLayerStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type ROLayerStore interface { + ROFileBasedStore + ROMetadataStore + ROLayerBigDataStore + Exists(id string) bool + Get(id string) (*Layer, error) + Status() ([][2]string, error) + Changes(from, to string) ([]archive.Change, error) + Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) + DiffSize(from, to string) (int64, error) + Size(name string) (int64, error) + Lookup(name string) (string, error) + LayersByCompressedDigest(d digest.Digest) ([]Layer, error) + LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + Layers() ([]Layer, error) +} + +// LayerStore is a deprecated interface with no documented way to use it from callers outside of c/storage. +// +// Deprecated: There is no way to use this from any external user of c/storage to invoke c/storage functionality. +type LayerStore interface { + ROLayerStore + RWFileBasedStore + RWMetadataStore + FlaggableStore + RWLayerBigDataStore + Create(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool) (*Layer, error) + CreateWithFlags(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}) (layer *Layer, err error) + Put(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) + SetNames(id string, names []string) error + AddNames(id string, names []string) error + RemoveNames(id string, names []string) error + Delete(id string) error + Wipe() error + Mount(id string, options drivers.MountOpts) (string, error) + Unmount(id string, force bool) (bool, error) + Mounted(id string) (int, error) + ParentOwners(id string) (uids, gids []int, err error) + ApplyDiff(to string, diff io.Reader) (int64, error) + ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) + CleanupStagingDirectory(stagingDirectory string) error + ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error + DifferTarget(id string) (string, error) + LoadLocked() error + PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) +} diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index 2642874b..e3847502 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -67,7 +67,7 @@ var ( const defaultPerms = os.FileMode(0555) func init() { - graphdriver.Register("aufs", Init) + graphdriver.MustRegister("aufs", Init) } // Driver contains information about the filesystem mounted. diff --git a/vendor/github.com/containers/storage/drivers/aufs/mount.go b/vendor/github.com/containers/storage/drivers/aufs/mount.go index 100e7537..156f4a4f 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/mount.go +++ b/vendor/github.com/containers/storage/drivers/aufs/mount.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package aufs diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 0b5d1c51..1d6d4666 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -42,7 +42,7 @@ import ( const defaultPerms = os.FileMode(0555) func init() { - graphdriver.Register("btrfs", Init) + graphdriver.MustRegister("btrfs", Init) } type btrfsOptions struct { diff --git a/vendor/github.com/containers/storage/drivers/btrfs/dummy_unsupported.go b/vendor/github.com/containers/storage/drivers/btrfs/dummy_unsupported.go index f0708888..c7d9d3b8 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/dummy_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/dummy_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux || !cgo // +build !linux !cgo package btrfs diff --git a/vendor/github.com/containers/storage/drivers/btrfs/version.go b/vendor/github.com/containers/storage/drivers/btrfs/version.go index edd8bdab..5816139f 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/version.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/version.go @@ -1,3 +1,4 @@ +//go:build linux && !btrfs_noversion && cgo // +build linux,!btrfs_noversion,cgo package btrfs diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index 34a76b82..ca43c3f0 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -115,7 +115,7 @@ func NewNaiveLayerIDMapUpdater(driver ProtoDriver) LayerIDMapUpdater { // on-disk owner UIDs and GIDs which are "host" values in the first map with // UIDs and GIDs for "host" values from the second map which correspond to the // same "container" IDs. -func (n *naiveLayerIDMapUpdater) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMappings, mountLabel string) error { +func (n *naiveLayerIDMapUpdater) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMappings, mountLabel string) (retErr error) { driver := n.ProtoDriver options := MountOpts{ MountLabel: mountLabel, @@ -124,9 +124,7 @@ func (n *naiveLayerIDMapUpdater) UpdateLayerIDMap(id string, toContainer, toHost if err != nil { return err } - defer func() { - driver.Put(id) - }() + defer driverPut(driver, id, &retErr) return ChownPathByMaps(layerFs, toContainer, toHost) } diff --git a/vendor/github.com/containers/storage/drivers/chown_darwin.go b/vendor/github.com/containers/storage/drivers/chown_darwin.go index a732075f..d6150cee 100644 --- a/vendor/github.com/containers/storage/drivers/chown_darwin.go +++ b/vendor/github.com/containers/storage/drivers/chown_darwin.go @@ -83,7 +83,7 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai uid, gid = mappedPair.UID, mappedPair.GID } if uid != int(st.Uid) || gid != int(st.Gid) { - cap, err := system.Lgetxattr(path, "security.capability") + capability, err := system.Lgetxattr(path, "security.capability") if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { return fmt.Errorf("%s: %w", os.Args[0], err) } @@ -98,8 +98,8 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai return fmt.Errorf("%s: %w", os.Args[0], err) } } - if cap != nil { - if err := system.Lsetxattr(path, "security.capability", cap, 0); err != nil { + if capability != nil { + if err := system.Lsetxattr(path, "security.capability", capability, 0); err != nil { return fmt.Errorf("%s: %w", os.Args[0], err) } } diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go index e97523c3..470e6a72 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go @@ -1,6 +1,7 @@ +//go:build !linux || !cgo // +build !linux !cgo -package copy +package copy //nolint: predeclared import ( "io" @@ -24,7 +25,7 @@ func DirCopy(srcDir, dstDir string, _ Mode, _ bool) error { } // CopyRegularToFile copies the content of a file to another -func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { +func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { //nolint: revive // "func name will be used as copy.CopyRegularToFile by other packages, and that stutters" f, err := os.Open(srcPath) if err != nil { return err @@ -35,6 +36,6 @@ func CopyRegularToFile(srcPath string, dstFile *os.File, fileinfo os.FileInfo, c } // CopyRegular copies the content of a file to another -func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { +func CopyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { //nolint:revive // "func name will be used as copy.CopyRegular by other packages, and that stutters" return chrootarchive.NewArchiver(nil).CopyWithTar(srcPath, dstPath) } diff --git a/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go b/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go index 418b9e61..f85fb947 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package devmapper diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index f9f775a5..27a58a9e 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -23,7 +23,7 @@ import ( const defaultPerms = os.FileMode(0555) func init() { - graphdriver.Register("devicemapper", Init) + graphdriver.MustRegister("devicemapper", Init) } // Driver contains the device set mounted and the home directory diff --git a/vendor/github.com/containers/storage/drivers/devmapper/mount.go b/vendor/github.com/containers/storage/drivers/devmapper/mount.go index 41e73faf..db903ca5 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/mount.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/mount.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package devmapper diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 7d96ebe5..e47d50a7 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -8,13 +8,12 @@ import ( "path/filepath" "strings" - "github.com/sirupsen/logrus" - "github.com/vbatts/tar-split/tar/storage" - "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" ) // FsMagic unsigned id of the filesystem in use. @@ -39,7 +38,7 @@ var ( ErrLayerUnknown = errors.New("unknown layer") ) -//CreateOpts contains optional arguments for Create() and CreateReadWrite() +// CreateOpts contains optional arguments for Create() and CreateReadWrite() // methods. type CreateOpts struct { MountLabel string @@ -53,8 +52,8 @@ type MountOpts struct { // Mount label is the MAC Labels to assign to mount point (SELINUX) MountLabel string // UidMaps & GidMaps are the User Namespace mappings to be assigned to content in the mount point - UidMaps []idtools.IDMap // nolint: golint - GidMaps []idtools.IDMap // nolint: golint + UidMaps []idtools.IDMap //nolint: golint,revive + GidMaps []idtools.IDMap //nolint: golint Options []string // Volatile specifies whether the container storage can be optimized @@ -279,6 +278,14 @@ func init() { drivers = make(map[string]InitFunc) } +// MustRegister registers an InitFunc for the driver, or panics. +// It is suitable for package’s init() sections. +func MustRegister(name string, initFunc InitFunc) { + if err := Register(name, initFunc); err != nil { + panic(fmt.Sprintf("failed to register containers/storage graph driver %q: %v", name, err)) + } +} + // Register registers an InitFunc for the driver. func Register(name string, initFunc InitFunc) error { if _, exists := drivers[name]; exists { @@ -405,3 +412,21 @@ func scanPriorDrivers(root string) map[string]bool { } return driversMap } + +// driverPut is driver.Put, but errors are handled either by updating mainErr or just logging. +// Typical usage: +// +// func …(…) (err error) { +// … +// defer driverPut(driver, id, &err) +// } +func driverPut(driver ProtoDriver, id string, mainErr *error) { + if err := driver.Put(id); err != nil { + err = fmt.Errorf("unmounting layer %s: %w", id, err) + if *mainErr == nil { + *mainErr = err + } else { + logrus.Errorf(err.Error()) + } + } +} diff --git a/vendor/github.com/containers/storage/drivers/driver_solaris.go b/vendor/github.com/containers/storage/drivers/driver_solaris.go index 174fa967..e3293091 100644 --- a/vendor/github.com/containers/storage/drivers/driver_solaris.go +++ b/vendor/github.com/containers/storage/drivers/driver_solaris.go @@ -1,3 +1,4 @@ +//go:build solaris && cgo // +build solaris,cgo package graphdriver diff --git a/vendor/github.com/containers/storage/drivers/driver_unsupported.go b/vendor/github.com/containers/storage/drivers/driver_unsupported.go index 3932c3ea..8119d9a6 100644 --- a/vendor/github.com/containers/storage/drivers/driver_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/driver_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux && !windows && !freebsd && !solaris && !darwin // +build !linux,!windows,!freebsd,!solaris,!darwin package graphdriver diff --git a/vendor/github.com/containers/storage/drivers/fsdiff.go b/vendor/github.com/containers/storage/drivers/fsdiff.go index 5022037d..6b2496ec 100644 --- a/vendor/github.com/containers/storage/drivers/fsdiff.go +++ b/vendor/github.com/containers/storage/drivers/fsdiff.go @@ -65,7 +65,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare defer func() { if err != nil { - driver.Put(id) + driverPut(driver, id, &err) } }() @@ -80,7 +80,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() - driver.Put(id) + driverPut(driver, id, &err) return err }), nil } @@ -90,7 +90,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare if err != nil { return nil, err } - defer driver.Put(parent) + defer driverPut(driver, parent, &err) changes, err := archive.ChangesDirs(layerFs, idMappings, parentFs, parentMappings) if err != nil { @@ -104,7 +104,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() - driver.Put(id) + driverPut(driver, id, &err) // NaiveDiffDriver compares file metadata with parent layers. Parent layers // are extracted from tar's with full second precision on modified time. @@ -117,7 +117,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. -func (gdw *NaiveDiffDriver) Changes(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) ([]archive.Change, error) { +func (gdw *NaiveDiffDriver) Changes(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) (_ []archive.Change, retErr error) { driver := gdw.ProtoDriver if idMappings == nil { @@ -134,7 +134,7 @@ func (gdw *NaiveDiffDriver) Changes(id string, idMappings *idtools.IDMappings, p if err != nil { return nil, err } - defer driver.Put(id) + defer driverPut(driver, id, &retErr) parentFs := "" @@ -147,7 +147,7 @@ func (gdw *NaiveDiffDriver) Changes(id string, idMappings *idtools.IDMappings, p if err != nil { return nil, err } - defer driver.Put(parent) + defer driverPut(driver, parent, &retErr) } return archive.ChangesDirs(layerFs, idMappings, parentFs, parentMappings) @@ -171,10 +171,10 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, options ApplyDiffOpts) if err != nil { return } - defer driver.Put(id) + defer driverPut(driver, id, &err) defaultForceMask := os.FileMode(0700) - var forceMask *os.FileMode = nil + var forceMask *os.FileMode // = nil if runtime.GOOS == "darwin" { forceMask = &defaultForceMask } @@ -224,7 +224,7 @@ func (gdw *NaiveDiffDriver) DiffSize(id string, idMappings *idtools.IDMappings, if err != nil { return } - defer driver.Put(id) + defer driverPut(driver, id, &err) return archive.ChangesSize(layerFs, changes), nil } diff --git a/vendor/github.com/containers/storage/drivers/overlay/idmapped_utils.go b/vendor/github.com/containers/storage/drivers/overlay/idmapped_utils.go index a7924ff3..0b7c868a 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/idmapped_utils.go +++ b/vendor/github.com/containers/storage/drivers/overlay/idmapped_utils.go @@ -6,6 +6,7 @@ package overlay import ( "fmt" "os" + "runtime" "syscall" "unsafe" @@ -112,7 +113,14 @@ func createIDMappedMount(source, target string, pid int) error { // createUsernsProcess forks the current process and creates a user namespace using the specified // mappings. It returns the pid of the new process. func createUsernsProcess(uidMaps []idtools.IDMap, gidMaps []idtools.IDMap) (int, func(), error) { - pid, _, err := syscall.Syscall6(uintptr(unix.SYS_CLONE), unix.CLONE_NEWUSER|uintptr(unix.SIGCHLD), 0, 0, 0, 0, 0) + var pid uintptr + var err syscall.Errno + + if runtime.GOARCH == "s390x" { + pid, _, err = syscall.Syscall6(uintptr(unix.SYS_CLONE), 0, unix.CLONE_NEWUSER|uintptr(unix.SIGCHLD), 0, 0, 0, 0) + } else { + pid, _, err = syscall.Syscall6(uintptr(unix.SYS_CLONE), unix.CLONE_NEWUSER|uintptr(unix.SIGCHLD), 0, 0, 0, 0, 0) + } if err != 0 { return -1, nil, err } diff --git a/vendor/github.com/containers/storage/drivers/overlay/mount.go b/vendor/github.com/containers/storage/drivers/overlay/mount.go index cf37f800..de47951d 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/mount.go +++ b/vendor/github.com/containers/storage/drivers/overlay/mount.go @@ -8,14 +8,16 @@ import ( "flag" "fmt" "os" + "path/filepath" "runtime" + "strings" "github.com/containers/storage/pkg/reexec" "golang.org/x/sys/unix" ) func init() { - reexec.Register("storage-mountfrom", mountFromMain) + reexec.Register("storage-mountfrom", mountOverlayFromMain) } func fatal(err error) { @@ -31,7 +33,7 @@ type mountOptions struct { Flag uint32 } -func mountFrom(dir, device, target, mType string, flags uintptr, label string) error { +func mountOverlayFrom(dir, device, target, mType string, flags uintptr, label string) error { options := &mountOptions{ Device: device, Target: target, @@ -67,7 +69,7 @@ func mountFrom(dir, device, target, mType string, flags uintptr, label string) e } // mountfromMain is the entry-point for storage-mountfrom on re-exec. -func mountFromMain() { +func mountOverlayFromMain() { runtime.LockOSThread() flag.Parse() @@ -77,11 +79,96 @@ func mountFromMain() { fatal(err) } - if err := os.Chdir(flag.Arg(0)); err != nil { + // Mount the arguments passed from the specified directory. Some of the + // paths mentioned in the values we pass to the kernel are relative to + // the specified directory. + homedir := flag.Arg(0) + if err := os.Chdir(homedir); err != nil { fatal(err) } - if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { + pageSize := unix.Getpagesize() + if len(options.Label) < pageSize { + if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { + fatal(err) + } + os.Exit(0) + } + + // Those arguments still took up too much space. Open the diff + // directories and use their descriptor numbers as lowers, using + // /proc/self/fd as the current directory. + + // Split out the various options, since we need to manipulate the + // paths, but we don't want to mess with other options. + var upperk, upperv, workk, workv, lowerk, lowerv, labelk, labelv, others string + for _, arg := range strings.Split(options.Label, ",") { + kv := strings.SplitN(arg, "=", 2) + switch kv[0] { + case "upperdir": + upperk = "upperdir=" + upperv = kv[1] + case "workdir": + workk = "workdir=" + workv = kv[1] + case "lowerdir": + lowerk = "lowerdir=" + lowerv = kv[1] + case "label": + labelk = "label=" + labelv = kv[1] + default: + if others == "" { + others = arg + } else { + others = others + "," + arg + } + } + } + + // Make sure upperdir, workdir, and the target are absolute paths. + if upperv != "" && !filepath.IsAbs(upperv) { + upperv = filepath.Join(homedir, upperv) + } + if workv != "" && !filepath.IsAbs(workv) { + workv = filepath.Join(homedir, workv) + } + if !filepath.IsAbs(options.Target) { + options.Target = filepath.Join(homedir, options.Target) + } + + // Get a descriptor for each lower, and use that descriptor's name as + // the new value for the list of lowers, because it's shorter. + if lowerv != "" { + lowers := strings.Split(lowerv, ":") + for i := range lowers { + lowerFd, err := unix.Open(lowers[i], unix.O_RDONLY, 0) + if err != nil { + fatal(err) + } + lowers[i] = fmt.Sprintf("%d", lowerFd) + } + lowerv = strings.Join(lowers, ":") + } + + // Reconstruct the Label field. + options.Label = upperk + upperv + "," + workk + workv + "," + lowerk + lowerv + "," + labelk + labelv + "," + others + options.Label = strings.ReplaceAll(options.Label, ",,", ",") + + // Okay, try this, if we managed to make the arguments fit. + var err error + if len(options.Label) < pageSize { + if err := os.Chdir("/proc/self/fd"); err != nil { + fatal(err) + } + err = unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label) + } else { + err = fmt.Errorf("cannot mount layer, mount data %q too large %d >= page size %d", options.Label, len(options.Label), pageSize) + } + + // Clean up. + if err != nil { + fmt.Fprintf(os.Stderr, "creating overlay mount to %s, mount_data=%q\n", options.Target, options.Label) fatal(err) } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 844d2c79..d08e82df 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -73,21 +73,23 @@ const ( // or root directory. Mounts are always done relative to root and // referencing the symbolic links in order to ensure the number of // lower directories can fit in a single page for making the mount -// syscall. A hard upper limit of 128 lower layers is enforced to ensure +// syscall. A hard upper limit of 500 lower layers is enforced to ensure // that mounts do not fail due to length. const ( linkDir = "l" lowerFile = "lower" - maxDepth = 128 + maxDepth = 500 // idLength represents the number of random characters // which can be used to create the unique link identifier // for every layer. If this value is too long then the // page size limit for the mount command may be exceeded. // The idLength should be selected such that following equation - // is true (512 is a buffer for label metadata). - // ((idLength + len(linkDir) + 1) * maxDepth) <= (pageSize - 512) + // is true (512 is a buffer for label metadata, 128 is the + // number of lowers we want to be able to use without having + // to use bind mounts to get all the way to the kernel limit). + // ((idLength + len(linkDir) + 1) * 128) <= (pageSize - 512) idLength = 26 ) @@ -140,8 +142,8 @@ var ( ) func init() { - graphdriver.Register("overlay", Init) - graphdriver.Register("overlay2", Init) + graphdriver.MustRegister("overlay", Init) + graphdriver.MustRegister("overlay2", Init) } func hasMetacopyOption(opts []string) bool { @@ -309,9 +311,11 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) if err != nil { return nil, err } - if fsName, ok := graphdriver.FsNames[fsMagic]; ok { - backingFs = fsName + fsName, ok := graphdriver.FsNames[fsMagic] + if !ok { + return nil, fmt.Errorf("filesystem type %#x reported for %s is not supported with 'overlay': %w", fsMagic, filepath.Dir(home), graphdriver.ErrIncompatibleFS) } + backingFs = fsName runhome := filepath.Join(options.RunRoot, filepath.Base(home)) rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) @@ -1374,27 +1378,9 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return "", errors.New("max depth exceeded") } - // absLowers is the list of lowers as absolute paths, which works well with additional stores. + // absLowers is the list of lowers as absolute paths. absLowers := []string{} - // relLowers is the list of lowers as paths relative to the driver's home directory. - relLowers := []string{} - // Check if $link/../diff{1-*} exist. If they do, add them, in order, as the front of the lowers - // lists that we're building. "diff" itself is the upper, so it won't be in the lists. - link, err := os.ReadFile(path.Join(dir, "link")) - if err != nil { - if !os.IsNotExist(err) { - return "", err - } - logrus.Warnf("Can't read parent link %q because it does not exist. Going through storage to recreate the missing links.", path.Join(dir, "link")) - if err := d.recreateSymlinks(); err != nil { - return "", fmt.Errorf("recreating the links: %w", err) - } - link, err = os.ReadFile(path.Join(dir, "link")) - if err != nil { - return "", err - } - } diffN := 1 perms := defaultPerms if d.options.forceMask != nil { @@ -1408,7 +1394,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } for err == nil { absLowers = append(absLowers, filepath.Join(dir, nameWithSuffix("diff", diffN))) - relLowers = append(relLowers, dumbJoin(linkDir, string(link), "..", nameWithSuffix("diff", diffN))) diffN++ st, err = os.Stat(filepath.Join(dir, nameWithSuffix("diff", diffN))) if err == nil && !permsKnown { @@ -1433,6 +1418,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO perms = os.FileMode(st2.Mode()) permsKnown = true } + l = lower break } lower = "" @@ -1457,12 +1443,10 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO lower = newpath } absLowers = append(absLowers, lower) - relLowers = append(relLowers, l) diffN = 1 _, err = os.Stat(dumbJoin(lower, "..", nameWithSuffix("diff", diffN))) for err == nil { absLowers = append(absLowers, dumbJoin(lower, "..", nameWithSuffix("diff", diffN))) - relLowers = append(relLowers, dumbJoin(l, "..", nameWithSuffix("diff", diffN))) diffN++ _, err = os.Stat(dumbJoin(lower, "..", nameWithSuffix("diff", diffN))) } @@ -1470,7 +1454,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO if len(absLowers) == 0 { absLowers = append(absLowers, path.Join(dir, "empty")) - relLowers = append(relLowers, path.Join(id, "empty")) } // user namespace requires this to move a directory from lower to upper. rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) @@ -1604,28 +1587,23 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return nil } } else if len(mountData) >= pageSize { - // Use relative paths and mountFrom when the mount data has exceeded - // the page size. The mount syscall fails if the mount data cannot - // fit within a page and relative links make the mount data much - // smaller at the expense of requiring a fork exec to chroot. + // Use mountFrom when the mount data has exceeded the page size. The mount syscall fails if + // the mount data cannot fit within a page and relative links make the mount data much + // smaller at the expense of requiring a fork exec to chdir(). workdir = path.Join(id, "work") - //FIXME: We need to figure out to get this to work with additional stores if readWrite { diffDir := path.Join(id, "diff") - opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), diffDir, workdir) + opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workdir) } else { - opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(relLowers, ":")) + opts = fmt.Sprintf("lowerdir=%s:%s", diffDir, strings.Join(absLowers, ":")) } if len(optsList) > 0 { opts = fmt.Sprintf("%s,%s", opts, strings.Join(optsList, ",")) } mountData = label.FormatMountLabel(opts, options.MountLabel) - if len(mountData) >= pageSize { - return "", fmt.Errorf("cannot mount layer, mount label %q too large %d >= page size %d", options.MountLabel, len(mountData), pageSize) - } mountFunc = func(source string, target string, mType string, flags uintptr, label string) error { - return mountFrom(d.home, source, target, mType, flags, label) + return mountOverlayFrom(d.home, source, target, mType, flags, label) } mountTarget = path.Join(id, "merged") } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go index 0b70a5d9..88bfbf9c 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_cgo.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package overlay diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go index 1cdac777..2a7a307a 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go @@ -1,3 +1,4 @@ +//go:build linux && !cgo // +build linux,!cgo package overlay diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go index 49af84a2..33b163a8 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package overlay diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota.go b/vendor/github.com/containers/storage/drivers/quota/projectquota.go index ed4c7eaa..d831d848 100644 --- a/vendor/github.com/containers/storage/drivers/quota/projectquota.go +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota.go @@ -51,6 +51,7 @@ struct fsxattr { */ import "C" import ( + "errors" "fmt" "math" "os" @@ -78,6 +79,7 @@ type Control struct { backingFsBlockDev string nextProjectID uint32 quotas map[string]uint32 + basePath string } // Attempt to generate a unigue projectid. Multiple directories @@ -158,20 +160,22 @@ func NewControl(basePath string) (*Control, error) { Size: 0, Inodes: 0, } - if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { - return nil, err - } q := Control{ backingFsBlockDev: backingFsBlockDev, nextProjectID: minProjectID + 1, quotas: make(map[string]uint32), + basePath: basePath, + } + + if err := q.setProjectQuota(minProjectID, quota); err != nil { + return nil, err } // // get first project id to be used for next container // - err = q.findNextProjectID(basePath) + err = q.findNextProjectID() if err != nil { return nil, err } @@ -204,11 +208,11 @@ func (q *Control) SetQuota(targetPath string, quota Quota) error { // set the quota limit for the container's project id // logrus.Debugf("SetQuota path=%s, size=%d, inodes=%d, projectID=%d", targetPath, quota.Size, quota.Inodes, projectID) - return setProjectQuota(q.backingFsBlockDev, projectID, quota) + return q.setProjectQuota(projectID, quota) } // setProjectQuota - set the quota for project id on xfs block device -func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { +func (q *Control) setProjectQuota(projectID uint32, quota Quota) error { var d C.fs_disk_quota_t d.d_version = C.FS_DQUOT_VERSION d.d_id = C.__u32(projectID) @@ -225,15 +229,35 @@ func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) er d.d_ino_softlimit = d.d_ino_hardlimit } - var cs = C.CString(backingFsBlockDev) + var cs = C.CString(q.backingFsBlockDev) defer C.free(unsafe.Pointer(cs)) - _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, - uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), - uintptr(unsafe.Pointer(&d)), 0, 0) - if errno != 0 { + runQuotactl := func() syscall.Errno { + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, + uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), + uintptr(unsafe.Pointer(&d)), 0, 0) + return errno + } + + errno := runQuotactl() + + // If the backingFsBlockDev does not exist any more then try to recreate it. + if errors.Is(errno, unix.ENOENT) { + if _, err := makeBackingFsDev(q.basePath); err != nil { + return fmt.Errorf( + "failed to recreate missing backingFsBlockDev %s for projid %d: %w", + q.backingFsBlockDev, projectID, err, + ) + } + + if errno := runQuotactl(); errno != 0 { + return fmt.Errorf("failed to set quota limit for projid %d on %s after backingFsBlockDev recreation: %w", + projectID, q.backingFsBlockDev, errno) + } + + } else if errno != 0 { return fmt.Errorf("failed to set quota limit for projid %d on %s: %w", - projectID, backingFsBlockDev, errno) + projectID, q.backingFsBlockDev, errno) } return nil @@ -332,16 +356,16 @@ func setProjectID(targetPath string, projectID uint32) error { // findNextProjectID - find the next project id to be used for containers // by scanning driver home directory to find used project ids -func (q *Control) findNextProjectID(home string) error { - files, err := os.ReadDir(home) +func (q *Control) findNextProjectID() error { + files, err := os.ReadDir(q.basePath) if err != nil { - return fmt.Errorf("read directory failed : %s", home) + return fmt.Errorf("read directory failed : %s", q.basePath) } for _, file := range files { if !file.IsDir() { continue } - path := filepath.Join(home, file.Name()) + path := filepath.Join(q.basePath, file.Name()) projid, err := getProjectID(path) if err != nil { return err diff --git a/vendor/github.com/containers/storage/drivers/register/register_aufs.go b/vendor/github.com/containers/storage/drivers/register/register_aufs.go index 7743dced..bbb9cb65 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_aufs.go +++ b/vendor/github.com/containers/storage/drivers/register/register_aufs.go @@ -1,3 +1,4 @@ +//go:build !exclude_graphdriver_aufs && linux // +build !exclude_graphdriver_aufs,linux package register diff --git a/vendor/github.com/containers/storage/drivers/register/register_btrfs.go b/vendor/github.com/containers/storage/drivers/register/register_btrfs.go index 40ff1cdd..425ebd79 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_btrfs.go +++ b/vendor/github.com/containers/storage/drivers/register/register_btrfs.go @@ -1,3 +1,4 @@ +//go:build !exclude_graphdriver_btrfs && linux // +build !exclude_graphdriver_btrfs,linux package register diff --git a/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go b/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go index cefe2e8c..a744eaea 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go +++ b/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go @@ -1,3 +1,4 @@ +//go:build !exclude_graphdriver_devicemapper && linux && cgo // +build !exclude_graphdriver_devicemapper,linux,cgo package register diff --git a/vendor/github.com/containers/storage/drivers/register/register_overlay.go b/vendor/github.com/containers/storage/drivers/register/register_overlay.go index 30e3b4d7..95b77b73 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_overlay.go +++ b/vendor/github.com/containers/storage/drivers/register/register_overlay.go @@ -1,3 +1,4 @@ +//go:build !exclude_graphdriver_overlay && linux && cgo // +build !exclude_graphdriver_overlay,linux,cgo package register diff --git a/vendor/github.com/containers/storage/drivers/register/register_zfs.go b/vendor/github.com/containers/storage/drivers/register/register_zfs.go index 4623e7f4..8e5788a4 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_zfs.go +++ b/vendor/github.com/containers/storage/drivers/register/register_zfs.go @@ -1,3 +1,4 @@ +//go:build (!exclude_graphdriver_zfs && linux) || (!exclude_graphdriver_zfs && freebsd) || solaris // +build !exclude_graphdriver_zfs,linux !exclude_graphdriver_zfs,freebsd solaris package register diff --git a/vendor/github.com/containers/storage/drivers/template.go b/vendor/github.com/containers/storage/drivers/template.go index d40d71cf..7b96c082 100644 --- a/vendor/github.com/containers/storage/drivers/template.go +++ b/vendor/github.com/containers/storage/drivers/template.go @@ -1,9 +1,8 @@ package graphdriver import ( - "github.com/sirupsen/logrus" - "github.com/containers/storage/pkg/idtools" + "github.com/sirupsen/logrus" ) // TemplateDriver is just barely enough of a driver that we can implement a diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go index 8ac80ee1..d94756bd 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package vfs // import "github.com/containers/storage/drivers/vfs" diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index b1073d55..50b4b281 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -28,7 +28,7 @@ var ( const defaultPerms = os.FileMode(0555) func init() { - graphdriver.Register("vfs", Init) + graphdriver.MustRegister("vfs", Init) } // Init returns a new VFS driver. @@ -98,7 +98,7 @@ func (d *Driver) Status() [][2]string { // Metadata is used for implementing the graphdriver.ProtoDriver interface. VFS does not currently have any meta data. func (d *Driver) Metadata(id string) (map[string]string, error) { - return nil, nil + return nil, nil //nolint: nilnil } // Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver. @@ -194,7 +194,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool if parent != "" { parentDir, err := d.Get(parent, graphdriver.MountOpts{}) if err != nil { - return fmt.Errorf("%s: %s", parent, err) + return fmt.Errorf("%s: %w", parent, err) } if err := dirCopy(parentDir, dir); err != nil { return err diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index 7def16cd..4cd9fa72 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -53,7 +53,7 @@ var ( // init registers the windows graph drivers to the register. func init() { - graphdriver.Register("windowsfilter", InitFilter) + graphdriver.MustRegister("windowsfilter", InitFilter) // DOCKER_WINDOWSFILTER_NOREEXEC allows for inline processing which makes // debugging issues in the re-exec codepath significantly easier. if os.Getenv("DOCKER_WINDOWSFILTER_NOREEXEC") != "" { diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index 0d400178..d1467824 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -33,7 +33,7 @@ type zfsOptions struct { const defaultPerms = os.FileMode(0555) func init() { - graphdriver.Register("zfs", Init) + graphdriver.MustRegister("zfs", Init) } // Logger returns a zfs logger implementation. diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs_unsupported.go b/vendor/github.com/containers/storage/drivers/zfs/zfs_unsupported.go index 643b169b..738b0ae1 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs_unsupported.go @@ -1,11 +1,4 @@ +//go:build !linux && !freebsd // +build !linux,!freebsd package zfs - -func checkRootdirFs(rootdir string) error { - return nil -} - -func getMountpoint(id string) string { - return id -} diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index c76a6c9f..81aef7c9 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -1,7 +1,6 @@ package storage import ( - "errors" "fmt" "os" "path/filepath" @@ -94,11 +93,17 @@ type Image struct { Flags map[string]interface{} `json:"flags,omitempty"` } -// ROImageStore provides bookkeeping for information about Images. -type ROImageStore interface { - ROFileBasedStore - ROMetadataStore - ROBigDataStore +// roImageStore provides bookkeeping for information about Images. +type roImageStore interface { + roMetadataStore + roBigDataStore + + // startReading makes sure the store is fresh, and locks it for reading. + // If this succeeds, the caller MUST call stopReading(). + startReading() error + + // stopReading releases locks obtained by startReading. + stopReading() // Exists checks if there is an image with the given ID or name. Exists(id string) bool @@ -106,10 +111,6 @@ type ROImageStore interface { // Get retrieves information about an image given an ID or name. Get(id string) (*Image, error) - // Lookup attempts to translate a name to an ID. Most methods do this - // implicitly. - Lookup(name string) (string, error) - // Images returns a slice enumerating the known images. Images() ([]Image, error) @@ -120,34 +121,29 @@ type ROImageStore interface { ByDigest(d digest.Digest) ([]*Image, error) } -// ImageStore provides bookkeeping for information about Images. -type ImageStore interface { - ROImageStore - RWFileBasedStore - RWMetadataStore - RWImageBigDataStore - FlaggableStore +// rwImageStore provides bookkeeping for information about Images. +type rwImageStore interface { + roImageStore + rwMetadataStore + rwImageBigDataStore + flaggableStore + + // startWriting makes sure the store is fresh, and locks it for writing. + // If this succeeds, the caller MUST call stopWriting(). + startWriting() error + + // stopWriting releases locks obtained by startWriting. + stopWriting() // Create creates an image that has a specified ID (or a random one) and // optional names, using the specified layer as its topmost (hopefully // read-only) layer. That layer can be referenced by multiple images. Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (*Image, error) - // SetNames replaces the list of names associated with an image with the - // supplied values. The values are expected to be valid normalized + // updateNames modifies names associated with an image based on (op, names). + // The values are expected to be valid normalized // named image references. - // Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. - SetNames(id string, names []string) error - - // AddNames adds the supplied values to the list of names associated with the image with - // the specified id. The values are expected to be valid normalized - // named image references. - AddNames(id string, names []string) error - - // RemoveNames removes the supplied values from the list of names associated with the image with - // the specified id. The values are expected to be valid normalized - // named image references. - RemoveNames(id string, names []string) error + updateNames(id string, names []string, op updateNameOperation) error // Delete removes the record of the image. Delete(id string) error @@ -157,7 +153,7 @@ type ImageStore interface { } type imageStore struct { - lockfile Locker + lockfile Locker // lockfile.IsReadWrite can be used to distinguish between read-write and read-only image stores. dir string images []*Image idindex *truncindex.TruncIndex @@ -197,6 +193,91 @@ func copyImageSlice(slice []*Image) []*Image { return nil } +// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +// +// This is an internal implementation detail of imageStore construction, every other caller +// should use startReading() instead. +func (r *imageStore) startWritingWithReload(canReload bool) error { + r.lockfile.Lock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if canReload { + if err := r.reloadIfChanged(true); err != nil { + return err + } + } + + succeeded = true + return nil +} + +// startWriting makes sure the store is fresh, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +func (r *imageStore) startWriting() error { + return r.startWritingWithReload(true) +} + +// stopWriting releases locks obtained by startWriting. +func (r *imageStore) stopWriting() { + r.lockfile.Unlock() +} + +// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading. +// If this succeeds, the caller MUST call stopReading(). +// +// This is an internal implementation detail of imageStore construction, every other caller +// should use startReading() instead. +func (r *imageStore) startReadingWithReload(canReload bool) error { + r.lockfile.RLock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if canReload { + if err := r.reloadIfChanged(false); err != nil { + return err + } + } + + succeeded = true + return nil +} + +// startReading makes sure the store is fresh, and locks it for reading. +// If this succeeds, the caller MUST call stopReading(). +func (r *imageStore) startReading() error { + return r.startReadingWithReload(true) +} + +// stopReading releases locks obtained by startReading. +func (r *imageStore) stopReading() { + r.lockfile.Unlock() +} + +// reloadIfChanged reloads the contents of the store from disk if it is changed. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *imageStore) reloadIfChanged(lockedForWriting bool) error { + r.loadMut.Lock() + defer r.loadMut.Unlock() + + modified, err := r.lockfile.Modified() + if err == nil && modified { + return r.load(lockedForWriting) + } + return err +} + func (r *imageStore) Images() ([]Image, error) { images := make([]Image, len(r.images)) for i := range r.images { @@ -257,49 +338,57 @@ func (i *Image) recomputeDigests() error { return nil } -func (r *imageStore) Load() error { +// load reloads the contents of the store from disk. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *imageStore) load(lockedForWriting bool) error { shouldSave := false rpath := r.imagespath() data, err := os.ReadFile(rpath) if err != nil && !os.IsNotExist(err) { return err } + images := []*Image{} - idlist := []string{} + if len(data) != 0 { + if err := json.Unmarshal(data, &images); err != nil { + return fmt.Errorf("loading %q: %w", rpath, err) + } + } + idlist := make([]string, 0, len(images)) ids := make(map[string]*Image) names := make(map[string]*Image) digests := make(map[digest.Digest][]*Image) - if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil { - idlist = make([]string, 0, len(images)) - for n, image := range images { - ids[image.ID] = images[n] - idlist = append(idlist, image.ID) - for _, name := range image.Names { - if conflict, ok := names[name]; ok { - r.removeName(conflict, name) - shouldSave = true - } + for n, image := range images { + ids[image.ID] = images[n] + idlist = append(idlist, image.ID) + for _, name := range image.Names { + if conflict, ok := names[name]; ok { + r.removeName(conflict, name) + shouldSave = true } - // Compute the digest list. - err = image.recomputeDigests() - if err != nil { - return fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err) - } - for _, name := range image.Names { - names[name] = image - } - for _, digest := range image.Digests { - list := digests[digest] - digests[digest] = append(list, image) - } - image.ReadOnly = !r.IsReadWrite() } + // Compute the digest list. + if err := image.recomputeDigests(); err != nil { + return fmt.Errorf("computing digests for image with ID %q (%v): %w", image.ID, image.Names, err) + } + for _, name := range image.Names { + names[name] = image + } + for _, digest := range image.Digests { + list := digests[digest] + digests[digest] = append(list, image) + } + image.ReadOnly = !r.lockfile.IsReadWrite() } - if shouldSave && (!r.IsReadWrite() || !r.Locked()) { + + if shouldSave && (!r.lockfile.IsReadWrite() || !lockedForWriting) { + // Eventually, the callers should be modified to retry with a write lock if IsReadWrite && !lockedForWriting, instead. return ErrDuplicateImageNames } r.images = images - r.idindex = truncindex.NewTruncIndex(idlist) + r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store. r.byid = ids r.byname = names r.bydigest = digests @@ -309,13 +398,13 @@ func (r *imageStore) Load() error { return nil } +// Save saves the contents of the store to disk. It should be called with +// the lock held, locked for writing. func (r *imageStore) Save() error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to modify the image store at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } - if !r.Locked() { - return errors.New("image store is not locked for writing") - } + r.lockfile.AssertLockedForWriting() rpath := r.imagespath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -324,11 +413,13 @@ func (r *imageStore) Save() error { if err != nil { return err } - defer r.Touch() - return ioutils.AtomicWriteFile(rpath, jdata, 0600) + if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil { + return err + } + return r.lockfile.Touch() } -func newImageStore(dir string) (ImageStore, error) { +func newImageStore(dir string) (rwImageStore, error) { if err := os.MkdirAll(dir, 0700); err != nil { return nil, err } @@ -336,8 +427,6 @@ func newImageStore(dir string) (ImageStore, error) { if err != nil { return nil, err } - lockfile.Lock() - defer lockfile.Unlock() istore := imageStore{ lockfile: lockfile, dir: dir, @@ -346,19 +435,21 @@ func newImageStore(dir string) (ImageStore, error) { byname: make(map[string]*Image), bydigest: make(map[digest.Digest][]*Image), } - if err := istore.Load(); err != nil { + if err := istore.startWritingWithReload(false); err != nil { + return nil, err + } + defer istore.stopWriting() + if err := istore.load(true); err != nil { return nil, err } return &istore, nil } -func newROImageStore(dir string) (ROImageStore, error) { +func newROImageStore(dir string) (roImageStore, error) { lockfile, err := GetROLockfile(filepath.Join(dir, "images.lock")) if err != nil { return nil, err } - lockfile.RLock() - defer lockfile.Unlock() istore := imageStore{ lockfile: lockfile, dir: dir, @@ -367,7 +458,11 @@ func newROImageStore(dir string) (ROImageStore, error) { byname: make(map[string]*Image), bydigest: make(map[digest.Digest][]*Image), } - if err := istore.Load(); err != nil { + if err := istore.startReadingWithReload(false); err != nil { + return nil, err + } + defer istore.stopReading() + if err := istore.load(false); err != nil { return nil, err } return &istore, nil @@ -386,7 +481,7 @@ func (r *imageStore) lookup(id string) (*Image, bool) { } func (r *imageStore) ClearFlag(id string, flag string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to clear flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } image, ok := r.lookup(id) @@ -398,7 +493,7 @@ func (r *imageStore) ClearFlag(id string, flag string) error { } func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to set flags on images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } image, ok := r.lookup(id) @@ -413,7 +508,7 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { } func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time, searchableDigest digest.Digest) (image *Image, err error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return nil, fmt.Errorf("not allowed to create new images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } if id == "" { @@ -455,7 +550,9 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c return nil, fmt.Errorf("validating digests for new image: %w", err) } r.images = append(r.images, image) - r.idindex.Add(id) + // This can only fail on duplicate IDs, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // Implementing recovery from an unlikely and unimportant failure here would be too risky. + _ = r.idindex.Add(id) r.byid[id] = image for _, name := range names { r.byname[name] = image @@ -498,7 +595,7 @@ func (r *imageStore) Metadata(id string) (string, error) { } func (r *imageStore) SetMetadata(id, metadata string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to modify image metadata at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } if image, ok := r.lookup(id); ok { @@ -516,21 +613,8 @@ func (i *Image) addNameToHistory(name string) { i.NamesHistory = dedupeNames(append([]string{name}, i.NamesHistory...)) } -// Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. -func (r *imageStore) SetNames(id string, names []string) error { - return r.updateNames(id, names, setNames) -} - -func (r *imageStore) AddNames(id string, names []string) error { - return r.updateNames(id, names, addNames) -} - -func (r *imageStore) RemoveNames(id string, names []string) error { - return r.updateNames(id, names, removeNames) -} - func (r *imageStore) updateNames(id string, names []string, op updateNameOperation) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to change image name assignments at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } image, ok := r.lookup(id) @@ -557,7 +641,7 @@ func (r *imageStore) updateNames(id string, names []string, op updateNameOperati } func (r *imageStore) Delete(id string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } image, ok := r.lookup(id) @@ -572,7 +656,9 @@ func (r *imageStore) Delete(id string) error { } } delete(r.byid, id) - r.idindex.Delete(id) + // This can only fail if the ID is already missing, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // The store’s Delete method is used on various paths to recover from failures, so this should be robust against partially missing data. + _ = r.idindex.Delete(id) for _, name := range image.Names { delete(r.byname, name) } @@ -608,13 +694,6 @@ func (r *imageStore) Get(id string) (*Image, error) { return nil, fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown) } -func (r *imageStore) Lookup(name string) (id string, err error) { - if image, ok := r.lookup(name); ok { - return image.ID, nil - } - return "", fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown) -} - func (r *imageStore) Exists(id string) bool { _, ok := r.lookup(id) return ok @@ -698,7 +777,7 @@ func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func if key == "" { return fmt.Errorf("can't set empty name for image big data item: %w", ErrInvalidBigDataName) } - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to save data items associated with images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } image, ok := r.lookup(id) @@ -779,7 +858,7 @@ func (r *imageStore) SetBigData(id, key string, data []byte, digestManifest func } func (r *imageStore) Wipe() error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to delete images at %q: %w", r.imagespath(), ErrStoreIsReadOnly) } ids := make([]string, 0, len(r.byid)) @@ -793,50 +872,3 @@ func (r *imageStore) Wipe() error { } return nil } - -func (r *imageStore) Lock() { - r.lockfile.Lock() -} - -func (r *imageStore) RecursiveLock() { - r.lockfile.RecursiveLock() -} - -func (r *imageStore) RLock() { - r.lockfile.RLock() -} - -func (r *imageStore) Unlock() { - r.lockfile.Unlock() -} - -func (r *imageStore) Touch() error { - return r.lockfile.Touch() -} - -func (r *imageStore) Modified() (bool, error) { - return r.lockfile.Modified() -} - -func (r *imageStore) IsReadWrite() bool { - return r.lockfile.IsReadWrite() -} - -func (r *imageStore) TouchedSince(when time.Time) bool { - return r.lockfile.TouchedSince(when) -} - -func (r *imageStore) Locked() bool { - return r.lockfile.Locked() -} - -func (r *imageStore) ReloadIfChanged() error { - r.loadMut.Lock() - defer r.loadMut.Unlock() - - modified, err := r.Modified() - if err == nil && modified { - return r.Load() - } - return err -} diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index c23f0b26..8053373f 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -26,7 +26,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" - "github.com/opencontainers/selinux/go-selinux/label" + "github.com/opencontainers/selinux/go-selinux" "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/archive/tar" "github.com/vbatts/tar-split/tar/asm" @@ -137,13 +137,19 @@ type DiffOptions struct { Compression *archive.Compression } -// ROLayerStore wraps a graph driver, adding the ability to refer to layers by +// roLayerStore wraps a graph driver, adding the ability to refer to layers by // name, and keeping track of parent-child relationships, along with a list of // all known layers. -type ROLayerStore interface { - ROFileBasedStore - ROMetadataStore - ROLayerBigDataStore +type roLayerStore interface { + roMetadataStore + roLayerBigDataStore + + // startReading makes sure the store is fresh, and locks it for reading. + // If this succeeds, the caller MUST call stopReading(). + startReading() error + + // stopReading releases locks obtained by startReading. + stopReading() // Exists checks if a layer with the specified name or ID is known. Exists(id string) bool @@ -177,10 +183,6 @@ type ROLayerStore interface { // found, it returns an error. Size(name string) (int64, error) - // Lookup attempts to translate a name to an ID. Most methods do this - // implicitly. - Lookup(name string) (string, error) - // LayersByCompressedDigest returns a slice of the layers with the // specified compressed digest value recorded for them. LayersByCompressedDigest(d digest.Digest) ([]Layer, error) @@ -193,15 +195,21 @@ type ROLayerStore interface { Layers() ([]Layer, error) } -// LayerStore wraps a graph driver, adding the ability to refer to layers by +// rwLayerStore wraps a graph driver, adding the ability to refer to layers by // name, and keeping track of parent-child relationships, along with a list of // all known layers. -type LayerStore interface { - ROLayerStore - RWFileBasedStore - RWMetadataStore - FlaggableStore - RWLayerBigDataStore +type rwLayerStore interface { + roLayerStore + rwMetadataStore + flaggableStore + rwLayerBigDataStore + + // startWriting makes sure the store is fresh, and locks it for writing. + // If this succeeds, the caller MUST call stopWriting(). + startWriting() error + + // stopWriting releases locks obtained by startWriting. + stopWriting() // Create creates a new layer, optionally giving it a specified ID rather than // a randomly-generated one, either inheriting data from another specified @@ -218,18 +226,8 @@ type LayerStore interface { // Put combines the functions of CreateWithFlags and ApplyDiff. Put(id string, parent *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) - // SetNames replaces the list of names associated with a layer with the - // supplied values. - // Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. - SetNames(id string, names []string) error - - // AddNames adds the supplied values to the list of names associated with the layer with the - // specified id. - AddNames(id string, names []string) error - - // RemoveNames remove the supplied values from the list of names associated with the layer with the - // specified id. - RemoveNames(id string, names []string) error + // updateNames modifies names associated with a layer based on (op, names). + updateNames(id string, names []string, op updateNameOperation) error // Delete deletes a layer with the specified name or ID. Delete(id string) error @@ -270,10 +268,6 @@ type LayerStore interface { // DifferTarget gets the location where files are stored for the layer. DifferTarget(id string) (string, error) - // LoadLocked wraps Load in a locked state. This means it loads the store - // and cleans-up invalid layers if needed. - LoadLocked() error - // PutAdditionalLayer creates a layer using the diff contained in the additional layer // store. // This API is experimental and can be changed without bumping the major version number. @@ -293,8 +287,6 @@ type layerStore struct { bymount map[string]*Layer bycompressedsum map[digest.Digest][]string byuncompressedsum map[digest.Digest][]string - uidMap []idtools.IDMap - gidMap []idtools.IDMap loadMut sync.Mutex layerspathModified time.Time } @@ -324,6 +316,125 @@ func copyLayer(l *Layer) *Layer { } } +// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +// +// This is an internal implementation detail of layerStore construction, every other caller +// should use startWriting() instead. +func (r *layerStore) startWritingWithReload(canReload bool) error { + r.lockfile.Lock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if canReload { + if err := r.reloadIfChanged(true); err != nil { + return err + } + } + + succeeded = true + return nil +} + +// startWriting makes sure the store is fresh, and locks it for writing. +// If this succeeds, the caller MUST call stopWriting(). +func (r *layerStore) startWriting() error { + return r.startWritingWithReload(true) +} + +// stopWriting releases locks obtained by startWriting. +func (r *layerStore) stopWriting() { + r.lockfile.Unlock() +} + +// startReadingWithReload makes sure the store is fresh if canReload, and locks it for reading. +// If this succeeds, the caller MUST call stopReading(). +// +// This is an internal implementation detail of layerStore construction, every other caller +// should use startReading() instead. +func (r *layerStore) startReadingWithReload(canReload bool) error { + r.lockfile.RLock() + succeeded := false + defer func() { + if !succeeded { + r.lockfile.Unlock() + } + }() + + if canReload { + if err := r.reloadIfChanged(false); err != nil { + return err + } + } + + succeeded = true + return nil +} + +// startReading makes sure the store is fresh, and locks it for reading. +// If this succeeds, the caller MUST call stopReading(). +func (r *layerStore) startReading() error { + return r.startReadingWithReload(true) +} + +// stopReading releases locks obtained by startReading. +func (r *layerStore) stopReading() { + r.lockfile.Unlock() +} + +// Modified() checks if the most recent writer was a party other than the +// last recorded writer. It should only be called with the lock held. +func (r *layerStore) Modified() (bool, error) { + var mmodified, tmodified bool + lmodified, err := r.lockfile.Modified() + if err != nil { + return lmodified, err + } + if r.lockfile.IsReadWrite() { + r.mountsLockfile.RLock() + defer r.mountsLockfile.Unlock() + mmodified, err = r.mountsLockfile.Modified() + if err != nil { + return lmodified, err + } + } + + if lmodified || mmodified { + return true, nil + } + + // If the layers.json file has been modified manually, then we have to + // reload the storage in any case. + info, err := os.Stat(r.layerspath()) + if err != nil && !os.IsNotExist(err) { + return false, fmt.Errorf("stat layers file: %w", err) + } + if info != nil { + tmodified = info.ModTime() != r.layerspathModified + } + + return tmodified, nil +} + +// reloadIfChanged reloads the contents of the store from disk if it is changed. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *layerStore) reloadIfChanged(lockedForWriting bool) error { + r.loadMut.Lock() + defer r.loadMut.Unlock() + + modified, err := r.Modified() + if err == nil && modified { + return r.load(lockedForWriting) + } + return err +} + func (r *layerStore) Layers() ([]Layer, error) { layers := make([]Layer, len(r.layers)) for i := range r.layers { @@ -340,7 +451,11 @@ func (r *layerStore) layerspath() string { return filepath.Join(r.layerdir, "layers.json") } -func (r *layerStore) Load() error { +// load reloads the contents of the store from disk. +// +// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true +// if it is held for writing. +func (r *layerStore) load(lockedForWriting bool) error { shouldSave := false rpath := r.layerspath() info, err := os.Stat(rpath) @@ -355,62 +470,67 @@ func (r *layerStore) Load() error { if err != nil && !os.IsNotExist(err) { return err } + layers := []*Layer{} - idlist := []string{} + if len(data) != 0 { + if err := json.Unmarshal(data, &layers); err != nil { + return fmt.Errorf("loading %q: %w", rpath, err) + } + } + idlist := make([]string, 0, len(layers)) ids := make(map[string]*Layer) names := make(map[string]*Layer) compressedsums := make(map[digest.Digest][]string) uncompressedsums := make(map[digest.Digest][]string) - if r.IsReadWrite() { - label.ClearLabels() + if r.lockfile.IsReadWrite() { + selinux.ClearLabels() } - if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil { - idlist = make([]string, 0, len(layers)) - for n, layer := range layers { - ids[layer.ID] = layers[n] - idlist = append(idlist, layer.ID) - for _, name := range layer.Names { - if conflict, ok := names[name]; ok { - r.removeName(conflict, name) - shouldSave = true - } - names[name] = layers[n] + for n, layer := range layers { + ids[layer.ID] = layers[n] + idlist = append(idlist, layer.ID) + for _, name := range layer.Names { + if conflict, ok := names[name]; ok { + r.removeName(conflict, name) + shouldSave = true } - if layer.CompressedDigest != "" { - compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID) - } - if layer.UncompressedDigest != "" { - uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID) - } - if layer.MountLabel != "" { - label.ReserveLabel(layer.MountLabel) - } - layer.ReadOnly = !r.IsReadWrite() + names[name] = layers[n] } - err = nil + if layer.CompressedDigest != "" { + compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID) + } + if layer.UncompressedDigest != "" { + uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID) + } + if layer.MountLabel != "" { + selinux.ReserveLabel(layer.MountLabel) + } + layer.ReadOnly = !r.lockfile.IsReadWrite() } - if shouldSave && (!r.IsReadWrite() || !r.Locked()) { + + if shouldSave && (!r.lockfile.IsReadWrite() || !lockedForWriting) { + // Eventually, the callers should be modified to retry with a write lock if IsReadWrite && !lockedForWriting, instead. return ErrDuplicateLayerNames } r.layers = layers - r.idindex = truncindex.NewTruncIndex(idlist) + r.idindex = truncindex.NewTruncIndex(idlist) // Invalid values in idlist are ignored: they are not a reason to refuse processing the whole store. r.byid = ids r.byname = names r.bycompressedsum = compressedsums r.byuncompressedsum = uncompressedsums // Load and merge information about which layers are mounted, and where. - if r.IsReadWrite() { + if r.lockfile.IsReadWrite() { r.mountsLockfile.RLock() defer r.mountsLockfile.Unlock() - if err = r.loadMounts(); err != nil { + if err := r.loadMounts(); err != nil { return err } // Last step: as we’re writable, try to remove anything that a previous // user of this storage area marked for deletion but didn't manage to // actually delete. - if r.Locked() { + var incompleteDeletionErrors error // = nil + if lockedForWriting { for _, layer := range r.layers { if layer.Flags == nil { layer.Flags = make(map[string]interface{}) @@ -419,24 +539,26 @@ func (r *layerStore) Load() error { logrus.Warnf("Found incomplete layer %#v, deleting it", layer.ID) err = r.deleteInternal(layer.ID) if err != nil { - break + // Don't return the error immediately, because deleteInternal does not saveLayers(); + // Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully + // deleted incomplete layers have their metadata correctly removed. + incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors, + fmt.Errorf("deleting layer %#v: %w", layer.ID, err)) } shouldSave = true } } } if shouldSave { - return r.saveLayers() + if err := r.saveLayers(); err != nil { + return err + } + } + if incompleteDeletionErrors != nil { + return incompleteDeletionErrors } } - - return err -} - -func (r *layerStore) LoadLocked() error { - r.lockfile.Lock() - defer r.lockfile.Unlock() - return r.Load() + return nil } func (r *layerStore) loadMounts() error { @@ -476,10 +598,11 @@ func (r *layerStore) loadMounts() error { return err } +// Save saves the contents of the store to disk. It should be called with +// the lock held, locked for writing. func (r *layerStore) Save() error { r.mountsLockfile.Lock() defer r.mountsLockfile.Unlock() - defer r.mountsLockfile.Touch() if err := r.saveLayers(); err != nil { return err } @@ -487,12 +610,10 @@ func (r *layerStore) Save() error { } func (r *layerStore) saveLayers() error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } - if !r.Locked() { - return errors.New("layer store is not locked for writing") - } + r.lockfile.AssertLockedForWriting() rpath := r.layerspath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -501,17 +622,17 @@ func (r *layerStore) saveLayers() error { if err != nil { return err } - defer r.Touch() - return ioutils.AtomicWriteFile(rpath, jldata, 0600) + if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil { + return err + } + return r.lockfile.Touch() } func (r *layerStore) saveMounts() error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to modify the layer store at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } - if !r.mountsLockfile.Locked() { - return errors.New("layer store mount information is not locked for writing") - } + r.mountsLockfile.AssertLockedForWriting() mpath := r.mountspath() if err := os.MkdirAll(filepath.Dir(mpath), 0700); err != nil { return err @@ -533,10 +654,13 @@ func (r *layerStore) saveMounts() error { if err = ioutils.AtomicWriteFile(mpath, jmdata, 0600); err != nil { return err } + if err := r.mountsLockfile.Touch(); err != nil { + return err + } return r.loadMounts() } -func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Driver) (LayerStore, error) { +func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Driver) (rwLayerStore, error) { if err := os.MkdirAll(rundir, 0700); err != nil { return nil, err } @@ -560,18 +684,18 @@ func (s *store) newLayerStore(rundir string, layerdir string, driver drivers.Dri byid: make(map[string]*Layer), bymount: make(map[string]*Layer), byname: make(map[string]*Layer), - uidMap: copyIDMap(s.uidMap), - gidMap: copyIDMap(s.gidMap), } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.Load(); err != nil { + if err := rlstore.startWritingWithReload(false); err != nil { + return nil, err + } + defer rlstore.stopWriting() + if err := rlstore.load(true); err != nil { return nil, err } return &rlstore, nil } -func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (ROLayerStore, error) { +func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (roLayerStore, error) { lockfile, err := GetROLockfile(filepath.Join(layerdir, "layers.lock")) if err != nil { return nil, err @@ -586,9 +710,11 @@ func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (ROL bymount: make(map[string]*Layer), byname: make(map[string]*Layer), } - rlstore.RLock() - defer rlstore.Unlock() - if err := rlstore.Load(); err != nil { + if err := rlstore.startReadingWithReload(false); err != nil { + return nil, err + } + defer rlstore.stopReading() + if err := rlstore.load(false); err != nil { return nil, err } return &rlstore, nil @@ -621,7 +747,7 @@ func (r *layerStore) Size(name string) (int64, error) { } func (r *layerStore) ClearFlag(id string, flag string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to clear flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } layer, ok := r.lookup(id) @@ -633,7 +759,7 @@ func (r *layerStore) ClearFlag(id string, flag string) error { } func (r *layerStore) SetFlag(id string, flag string, value interface{}) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to set flags on layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } layer, ok := r.lookup(id) @@ -685,7 +811,9 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s // TODO: check if necessary fields are filled r.layers = append(r.layers, layer) - r.idindex.Add(id) + // This can only fail on duplicate IDs, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // Implementing recovery from an unlikely and unimportant failure here would be too risky. + _ = r.idindex.Add(id) r.byid[id] = layer for _, name := range names { // names got from the additional layer store won't be used r.byname[name] = layer @@ -697,14 +825,16 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID) } if err := r.Save(); err != nil { - r.driver.Remove(id) + if err2 := r.driver.Remove(id); err2 != nil { + logrus.Errorf("While recovering from a failure to save layers, error deleting layer %#v: %v", id, err2) + } return nil, err } return copyLayer(layer), nil } func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (*Layer, int64, error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return nil, -1, fmt.Errorf("not allowed to create new layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } if err := os.MkdirAll(r.rundir, 0700); err != nil { @@ -770,7 +900,7 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab parentMappings = &idtools.IDMappings{} } if mountLabel != "" { - label.ReserveLabel(mountLabel) + selinux.ReserveLabel(mountLabel) } // Before actually creating the layer, make a persistent record of it with incompleteFlag, @@ -795,7 +925,9 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab BigDataNames: []string{}, } r.layers = append(r.layers, layer) - r.idindex.Add(id) + // This can only fail if the ID is already missing, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // This is on various paths to recover from failures, so this should be robust against partially missing data. + _ = r.idindex.Add(id) r.byid[id] = layer for _, name := range names { r.byname[name] = layer @@ -907,7 +1039,7 @@ func (r *layerStore) Create(id string, parent *Layer, names []string, mountLabel } func (r *layerStore) Mounted(id string) (int, error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return 0, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly) } r.mountsLockfile.RLock() @@ -937,7 +1069,7 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error) // You are not allowed to mount layers from readonly stores if they // are not mounted read/only. - if !r.IsReadWrite() && !hasReadOnlyOpt(options.Options) { + if !r.lockfile.IsReadWrite() && !hasReadOnlyOpt(options.Options) { return "", fmt.Errorf("not allowed to update mount locations for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly) } r.mountsLockfile.Lock() @@ -947,7 +1079,6 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error) return "", err } } - defer r.mountsLockfile.Touch() layer, ok := r.lookup(id) if !ok { return "", ErrLayerUnknown @@ -988,7 +1119,7 @@ func (r *layerStore) Mount(id string, options drivers.MountOpts) (string, error) } func (r *layerStore) Unmount(id string, force bool) (bool, error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return false, fmt.Errorf("not allowed to update mount locations for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly) } r.mountsLockfile.Lock() @@ -998,7 +1129,6 @@ func (r *layerStore) Unmount(id string, force bool) (bool, error) { return false, err } } - defer r.mountsLockfile.Touch() layer, ok := r.lookup(id) if !ok { layerByMount, ok := r.bymount[filepath.Clean(id)] @@ -1027,7 +1157,7 @@ func (r *layerStore) Unmount(id string, force bool) (bool, error) { } func (r *layerStore) ParentOwners(id string) (uids, gids []int, err error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return nil, nil, fmt.Errorf("no mount information for layers at %q: %w", r.mountspath(), ErrStoreIsReadOnly) } r.mountsLockfile.RLock() @@ -1101,21 +1231,8 @@ func (r *layerStore) removeName(layer *Layer, name string) { layer.Names = stringSliceWithoutValue(layer.Names, name) } -// Deprecated: Prone to race conditions, suggested alternatives are `AddNames` and `RemoveNames`. -func (r *layerStore) SetNames(id string, names []string) error { - return r.updateNames(id, names, setNames) -} - -func (r *layerStore) AddNames(id string, names []string) error { - return r.updateNames(id, names, addNames) -} - -func (r *layerStore) RemoveNames(id string, names []string) error { - return r.updateNames(id, names, removeNames) -} - func (r *layerStore) updateNames(id string, names []string, op updateNameOperation) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to change layer name assignments at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } layer, ok := r.lookup(id) @@ -1163,7 +1280,7 @@ func (r *layerStore) SetBigData(id, key string, data io.Reader) error { if key == "" { return fmt.Errorf("can't set empty name for layer big data item: %w", ErrInvalidBigDataName) } - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to save data items associated with layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } layer, ok := r.lookup(id) @@ -1222,7 +1339,7 @@ func (r *layerStore) Metadata(id string) (string, error) { } func (r *layerStore) SetMetadata(id, metadata string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to modify layer metadata at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } if layer, ok := r.lookup(id); ok { @@ -1248,7 +1365,7 @@ func layerHasIncompleteFlag(layer *Layer) bool { } func (r *layerStore) deleteInternal(id string) error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } layer, ok := r.lookup(id) @@ -1268,8 +1385,7 @@ func (r *layerStore) deleteInternal(id string) error { // We never unset incompleteFlag; below, we remove the entire object from r.layers. id = layer.ID - err := r.driver.Remove(id) - if err != nil { + if err := r.driver.Remove(id); err != nil { return err } @@ -1279,7 +1395,9 @@ func (r *layerStore) deleteInternal(id string) error { for _, name := range layer.Names { delete(r.byname, name) } - r.idindex.Delete(id) + // This can only fail if the ID is already missing, which shouldn’t happen — and in that case the index is already in the desired state anyway. + // The store’s Delete method is used on various paths to recover from failures, so this should be robust against partially missing data. + _ = r.idindex.Delete(id) mountLabel := layer.MountLabel if layer.MountPoint != "" { delete(r.bymount, layer.MountPoint) @@ -1309,7 +1427,7 @@ func (r *layerStore) deleteInternal(id string) error { } } if !found { - label.ReleaseLabel(mountLabel) + selinux.ReleaseLabel(mountLabel) } } @@ -1365,13 +1483,6 @@ func (r *layerStore) Delete(id string) error { return r.Save() } -func (r *layerStore) Lookup(name string) (id string, err error) { - if layer, ok := r.lookup(name); ok { - return layer.ID, nil - } - return "", ErrLayerUnknown -} - func (r *layerStore) Exists(id string) bool { _, ok := r.lookup(id) return ok @@ -1385,7 +1496,7 @@ func (r *layerStore) Get(id string) (*Layer, error) { } func (r *layerStore) Wipe() error { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return fmt.Errorf("not allowed to delete layers at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } ids := make([]string, 0, len(r.byid)) @@ -1472,6 +1583,24 @@ func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) { }, nil } +// writeCompressedData copies data from source to compressor, which is on top of pwriter. +func writeCompressedData(compressor io.WriteCloser, source io.ReadCloser) error { + defer compressor.Close() + defer source.Close() + _, err := io.Copy(compressor, source) + return err +} + +// writeCompressedDataGoroutine copies data from source to compressor, which is on top of pwriter. +// All error must be reported by updating pwriter. +func writeCompressedDataGoroutine(pwriter *io.PipeWriter, compressor io.WriteCloser, source io.ReadCloser) { + err := errors.New("internal error: unexpected panic in writeCompressedDataGoroutine") + defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily. + _ = pwriter.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil + }() + err = writeCompressedData(compressor, source) +} + func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { var metadata storage.Unpacker @@ -1503,12 +1632,7 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, preader.Close() return nil, err } - go func() { - defer pwriter.Close() - defer compressor.Close() - defer rc.Close() - io.Copy(compressor, rc) - }() + go writeCompressedDataGoroutine(pwriter, compressor, rc) return preader, nil } @@ -1637,7 +1761,7 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error } func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, diff io.Reader) (size int64, err error) { - if !r.IsReadWrite() { + if !r.lockfile.IsReadWrite() { return -1, fmt.Errorf("not allowed to modify layer contents at %q: %w", r.layerspath(), ErrStoreIsReadOnly) } @@ -1724,13 +1848,11 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, return -1, err } compressor.Close() - if err == nil { - if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil { - return -1, err - } - if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0600); err != nil { - return -1, err - } + if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil { + return -1, err + } + if err := ioutils.AtomicWriteFile(r.tspath(layer.ID), tsdata.Bytes(), 0600); err != nil { + return -1, err } if compressedDigester != nil { compressedDigest = compressedDigester.Digest() @@ -1825,7 +1947,9 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, } for k, v := range diffOutput.BigData { if err := r.SetBigData(id, k, bytes.NewReader(v)); err != nil { - r.Delete(id) + if err2 := r.Delete(id); err2 != nil { + logrus.Errorf("While recovering from a failure to set big data, error deleting layer %#v: %v", id, err2) + } return err } } @@ -1891,81 +2015,6 @@ func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error return r.layersByDigestMap(r.byuncompressedsum, d) } -func (r *layerStore) Lock() { - r.lockfile.Lock() -} - -func (r *layerStore) RecursiveLock() { - r.lockfile.RecursiveLock() -} - -func (r *layerStore) RLock() { - r.lockfile.RLock() -} - -func (r *layerStore) Unlock() { - r.lockfile.Unlock() -} - -func (r *layerStore) Touch() error { - return r.lockfile.Touch() -} - -func (r *layerStore) Modified() (bool, error) { - var mmodified, tmodified bool - lmodified, err := r.lockfile.Modified() - if err != nil { - return lmodified, err - } - if r.IsReadWrite() { - r.mountsLockfile.RLock() - defer r.mountsLockfile.Unlock() - mmodified, err = r.mountsLockfile.Modified() - if err != nil { - return lmodified, err - } - } - - if lmodified || mmodified { - return true, nil - } - - // If the layers.json file has been modified manually, then we have to - // reload the storage in any case. - info, err := os.Stat(r.layerspath()) - if err != nil && !os.IsNotExist(err) { - return false, fmt.Errorf("stat layers file: %w", err) - } - if info != nil { - tmodified = info.ModTime() != r.layerspathModified - } - - return tmodified, nil -} - -func (r *layerStore) IsReadWrite() bool { - return r.lockfile.IsReadWrite() -} - -func (r *layerStore) TouchedSince(when time.Time) bool { - return r.lockfile.TouchedSince(when) -} - -func (r *layerStore) Locked() bool { - return r.lockfile.Locked() -} - -func (r *layerStore) ReloadIfChanged() error { - r.loadMut.Lock() - defer r.loadMut.Unlock() - - modified, err := r.Modified() - if err == nil && modified { - return r.Load() - } - return err -} - func closeAll(closes ...func() error) (rErr error) { for _, f := range closes { if err := f(); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 82c0adeb..6209205b 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -527,6 +527,9 @@ func (ta *tarAppender) addTarFile(path, name string) error { if err := ReadUserXattrToTarHeader(path, hdr); err != nil { return err } + if err := ReadFileFlagsToTarHeader(path, hdr); err != nil { + return err + } if ta.CopyPass { copyPassHeader(hdr) } @@ -770,6 +773,15 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L } + // We defer setting flags on directories until the end of + // Unpack or UnpackLayer in case setting them makes the + // directory immutable. + if hdr.Typeflag != tar.TypeDir { + if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil { + return err + } + } + if len(errs) > 0 { logrus.WithFields(logrus.Fields{ "errors": errs, @@ -864,7 +876,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) rebaseName := options.RebaseNames[include] walkRoot := getWalkRoot(srcPath, include) - filepath.WalkDir(walkRoot, func(filePath string, d fs.DirEntry, err error) error { + if err := filepath.WalkDir(walkRoot, func(filePath string, d fs.DirEntry, err error) error { if err != nil { logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) return nil @@ -874,7 +886,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) if err != nil || (!options.IncludeSourceDir && relFilePath == "." && d.IsDir()) { // Error getting relative path OR we are looking // at the source directory path. Skip in both situations. - return nil + return nil //nolint: nilerr } if options.IncludeSourceDir && include == "." && relFilePath != "." { @@ -891,8 +903,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) if include != relFilePath { matches, err := pm.IsMatch(relFilePath) if err != nil { - logrus.Errorf("Matching %s: %v", relFilePath, err) - return err + return fmt.Errorf("matching %s: %w", relFilePath, err) } skip = matches } @@ -955,7 +966,10 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } } return nil - }) + }); err != nil { + logrus.Errorf("%s", err) + return + } } }() @@ -1099,6 +1113,9 @@ loop: if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { return err } + if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil { + return err + } } return nil } diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_110.go b/vendor/github.com/containers/storage/pkg/archive/archive_110.go index 7bc44a56..eab9da51 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_110.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_110.go @@ -1,3 +1,4 @@ +//go:build go1.10 // +build go1.10 package archive diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_19.go b/vendor/github.com/containers/storage/pkg/archive/archive_19.go index d19811fd..f591bf38 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_19.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_19.go @@ -1,3 +1,4 @@ +//go:build !go1.10 // +build !go1.10 package archive diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go b/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go new file mode 100644 index 00000000..4d362f07 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/archive_bsd.go @@ -0,0 +1,19 @@ +//go:build freebsd || darwin +// +build freebsd darwin + +package archive + +import ( + "archive/tar" + "os" + + "golang.org/x/sys/unix" +) + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo, forceMask *os.FileMode) error { + permissionsMask := hdrInfo.Mode() + if forceMask != nil { + permissionsMask = *forceMask + } + return unix.Fchmodat(unix.AT_FDCWD, path, uint32(permissionsMask), unix.AT_SYMLINK_NOFOLLOW) +} diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go b/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go deleted file mode 100644 index 36017c3b..00000000 --- a/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go +++ /dev/null @@ -1,129 +0,0 @@ -//go:build freebsd -// +build freebsd - -package archive - -import ( - "archive/tar" - "errors" - "os" - "path/filepath" - "syscall" - "unsafe" - - "github.com/containers/storage/pkg/idtools" - "github.com/containers/storage/pkg/system" - "github.com/containers/storage/pkg/unshare" - "golang.org/x/sys/unix" -) - -// fixVolumePathPrefix does platform specific processing to ensure that if -// the path being passed in is not in a volume path format, convert it to one. -func fixVolumePathPrefix(srcPath string) string { - return srcPath -} - -// getWalkRoot calculates the root path when performing a TarWithOptions. -// We use a separate function as this is platform specific. On Linux, we -// can't use filepath.Join(srcPath,include) because this will clean away -// a trailing "." or "/" which may be important. -func getWalkRoot(srcPath string, include string) string { - return srcPath + string(filepath.Separator) + include -} - -// CanonicalTarNameForPath returns platform-specific filepath -// to canonical posix-style path for tar archival. p is relative -// path. -func CanonicalTarNameForPath(p string) (string, error) { - return p, nil // already unix-style -} - -// chmodTarEntry is used to adjust the file permissions used in tar header based -// on the platform the archival is done. -func chmodTarEntry(perm os.FileMode) os.FileMode { - return perm // noop for unix as golang APIs provide perm bits correctly -} - -func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) { - s, ok := stat.(*syscall.Stat_t) - - if ok { - // Currently go does not fill in the major/minors - if s.Mode&unix.S_IFBLK != 0 || - s.Mode&unix.S_IFCHR != 0 { - hdr.Devmajor = int64(major(uint64(s.Rdev))) // nolint: unconvert - hdr.Devminor = int64(minor(uint64(s.Rdev))) // nolint: unconvert - } - } - - return -} - -func getInodeFromStat(stat interface{}) (inode uint64, err error) { - s, ok := stat.(*syscall.Stat_t) - - if ok { - inode = s.Ino - } - - return -} - -func getFileUIDGID(stat interface{}) (idtools.IDPair, error) { - s, ok := stat.(*syscall.Stat_t) - - if !ok { - return idtools.IDPair{}, errors.New("cannot convert stat value to syscall.Stat_t") - } - return idtools.IDPair{UID: int(s.Uid), GID: int(s.Gid)}, nil -} - -func major(device uint64) uint64 { - return (device >> 8) & 0xfff -} - -func minor(device uint64) uint64 { - return (device & 0xff) | ((device >> 12) & 0xfff00) -} - -// handleTarTypeBlockCharFifo is an OS-specific helper function used by -// createTarFile to handle the following types of header: Block; Char; Fifo -func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { - if unshare.IsRootless() { - // cannot create a device if running in user namespace - return nil - } - - mode := uint32(hdr.Mode & 07777) - switch hdr.Typeflag { - case tar.TypeBlock: - mode |= unix.S_IFBLK - case tar.TypeChar: - mode |= unix.S_IFCHR - case tar.TypeFifo: - mode |= unix.S_IFIFO - } - - return system.Mknod(path, mode, uint64(system.Mkdev(hdr.Devmajor, hdr.Devminor))) -} - -func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo, forceMask *os.FileMode) error { - permissionsMask := hdrInfo.Mode() - if forceMask != nil { - permissionsMask = *forceMask - } - p, err := unix.BytePtrFromString(path) - if err != nil { - return err - } - _, _, e1 := unix.Syscall(unix.SYS_LCHMOD, uintptr(unsafe.Pointer(p)), uintptr(permissionsMask), 0) - if e1 != 0 { - return e1 - } - return nil -} - -// Hardlink without following symlinks -func handleLLink(targetPath string, path string) error { - return unix.Linkat(unix.AT_FDCWD, targetPath, unix.AT_FDCWD, path, 0) -} diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go index 51fbd9a2..36e5d4bc 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go @@ -189,3 +189,22 @@ func GetFileOwner(path string) (uint32, uint32, uint32, error) { } return 0, 0, uint32(f.Mode()), nil } + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo, forceMask *os.FileMode) error { + permissionsMask := hdrInfo.Mode() + if forceMask != nil { + permissionsMask = *forceMask + } + if hdr.Typeflag == tar.TypeLink { + if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { + if err := os.Chmod(path, permissionsMask); err != nil { + return err + } + } + } else if hdr.Typeflag != tar.TypeSymlink { + if err := os.Chmod(path, permissionsMask); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_other.go b/vendor/github.com/containers/storage/pkg/archive/archive_other.go index 4b883444..2468ab3c 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_other.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_other.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package archive diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_unix.go b/vendor/github.com/containers/storage/pkg/archive/archive_unix.go index d0fb3306..f8a34c83 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_unix.go @@ -1,5 +1,5 @@ -//go:build !windows && !freebsd -// +build !windows,!freebsd +//go:build !windows +// +build !windows package archive @@ -50,8 +50,8 @@ func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) ( // Currently go does not fill in the major/minors if s.Mode&unix.S_IFBLK != 0 || s.Mode&unix.S_IFCHR != 0 { - hdr.Devmajor = int64(major(uint64(s.Rdev))) // nolint: unconvert - hdr.Devminor = int64(minor(uint64(s.Rdev))) // nolint: unconvert + hdr.Devmajor = int64(major(uint64(s.Rdev))) //nolint: unconvert + hdr.Devminor = int64(minor(uint64(s.Rdev))) //nolint: unconvert } } @@ -101,25 +101,6 @@ func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { return system.Mknod(path, mode, system.Mkdev(hdr.Devmajor, hdr.Devminor)) } -func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo, forceMask *os.FileMode) error { - permissionsMask := hdrInfo.Mode() - if forceMask != nil { - permissionsMask = *forceMask - } - if hdr.Typeflag == tar.TypeLink { - if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { - if err := os.Chmod(path, permissionsMask); err != nil { - return err - } - } - } else if hdr.Typeflag != tar.TypeSymlink { - if err := os.Chmod(path, permissionsMask); err != nil { - return err - } - } - return nil -} - // Hardlink without symlinks func handleLLink(targetPath, path string) error { // Note: on Linux, the link syscall will not follow symlinks. diff --git a/vendor/github.com/containers/storage/pkg/archive/changes.go b/vendor/github.com/containers/storage/pkg/archive/changes.go index 6cd9e35e..fc705484 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes.go @@ -56,7 +56,7 @@ func (change *Change) String() string { return fmt.Sprintf("%s %s", change.Kind, change.Path) } -// for sort.Sort +// changesByPath implements sort.Interface. type changesByPath []Change func (c changesByPath) Less(i, j int) bool { return c[i].Path < c[j].Path } diff --git a/vendor/github.com/containers/storage/pkg/archive/changes_unix.go b/vendor/github.com/containers/storage/pkg/archive/changes_unix.go index 1cc1910f..6b2e5938 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes_unix.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package archive @@ -29,6 +30,7 @@ func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.Sta if oldStat.Mode() != newStat.Mode() || ownerChanged || oldStat.Rdev() != newStat.Rdev() || + oldStat.Flags() != newStat.Flags() || // Don't look at size for dirs, its not a good measure of change (oldStat.Mode()&unix.S_IFDIR != unix.S_IFDIR && (!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) { diff --git a/vendor/github.com/containers/storage/pkg/archive/copy_unix.go b/vendor/github.com/containers/storage/pkg/archive/copy_unix.go index e305b5e4..d6c5fd98 100644 --- a/vendor/github.com/containers/storage/pkg/archive/copy_unix.go +++ b/vendor/github.com/containers/storage/pkg/archive/copy_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package archive diff --git a/vendor/github.com/containers/storage/pkg/archive/diff.go b/vendor/github.com/containers/storage/pkg/archive/diff.go index 7e835d44..8fec5af3 100644 --- a/vendor/github.com/containers/storage/pkg/archive/diff.go +++ b/vendor/github.com/containers/storage/pkg/archive/diff.go @@ -145,6 +145,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, return nil } if _, exists := unpackedPaths[path]; !exists { + if err := resetImmutable(path, nil); err != nil { + return err + } err := os.RemoveAll(path) return err } @@ -156,6 +159,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, } else { originalBase := base[len(WhiteoutPrefix):] originalPath := filepath.Join(dir, originalBase) + if err := resetImmutable(originalPath, nil); err != nil { + return 0, err + } if err := os.RemoveAll(originalPath); err != nil { return 0, err } @@ -165,7 +171,15 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). + // + // We always reset the immutable flag (if present) to allow metadata + // changes and to allow directory modification. The flag will be + // re-applied based on the contents of hdr either at the end for + // directories or in createTarFile otherwise. if fi, err := os.Lstat(path); err == nil { + if err := resetImmutable(path, &fi); err != nil { + return 0, err + } if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return 0, err @@ -215,6 +229,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { return 0, err } + if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil { + return 0, err + } } return size, nil @@ -245,7 +262,9 @@ func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decomp if err != nil { return 0, err } - defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform + defer func() { + _, _ = system.Umask(oldmask) // Ignore err. This can only fail with ErrNotSupportedPlatform, in which case we would have failed above. + }() if decompress { layer, err = DecompressStream(layer) diff --git a/vendor/github.com/containers/storage/pkg/archive/fflags_bsd.go b/vendor/github.com/containers/storage/pkg/archive/fflags_bsd.go new file mode 100644 index 00000000..14661c41 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/fflags_bsd.go @@ -0,0 +1,167 @@ +//go:build freebsd +// +build freebsd + +package archive + +import ( + "archive/tar" + "fmt" + "math/bits" + "os" + "strings" + "syscall" + + "github.com/containers/storage/pkg/system" +) + +const ( + paxSCHILYFflags = "SCHILY.fflags" +) + +var ( + flagNameToValue = map[string]uint32{ + "sappnd": system.SF_APPEND, + "sappend": system.SF_APPEND, + "arch": system.SF_ARCHIVED, + "archived": system.SF_ARCHIVED, + "schg": system.SF_IMMUTABLE, + "schange": system.SF_IMMUTABLE, + "simmutable": system.SF_IMMUTABLE, + "sunlnk": system.SF_NOUNLINK, + "sunlink": system.SF_NOUNLINK, + "snapshot": system.SF_SNAPSHOT, + "uappnd": system.UF_APPEND, + "uappend": system.UF_APPEND, + "uarch": system.UF_ARCHIVE, + "uarchive": system.UF_ARCHIVE, + "hidden": system.UF_HIDDEN, + "uhidden": system.UF_HIDDEN, + "uchg": system.UF_IMMUTABLE, + "uchange": system.UF_IMMUTABLE, + "uimmutable": system.UF_IMMUTABLE, + "uunlnk": system.UF_NOUNLINK, + "uunlink": system.UF_NOUNLINK, + "offline": system.UF_OFFLINE, + "uoffline": system.UF_OFFLINE, + "opaque": system.UF_OPAQUE, + "rdonly": system.UF_READONLY, + "urdonly": system.UF_READONLY, + "readonly": system.UF_READONLY, + "ureadonly": system.UF_READONLY, + "reparse": system.UF_REPARSE, + "ureparse": system.UF_REPARSE, + "sparse": system.UF_SPARSE, + "usparse": system.UF_SPARSE, + "system": system.UF_SYSTEM, + "usystem": system.UF_SYSTEM, + } + // Only include the short names for the reverse map + flagValueToName = map[uint32]string{ + system.SF_APPEND: "sappnd", + system.SF_ARCHIVED: "arch", + system.SF_IMMUTABLE: "schg", + system.SF_NOUNLINK: "sunlnk", + system.SF_SNAPSHOT: "snapshot", + system.UF_APPEND: "uappnd", + system.UF_ARCHIVE: "uarch", + system.UF_HIDDEN: "hidden", + system.UF_IMMUTABLE: "uchg", + system.UF_NOUNLINK: "uunlnk", + system.UF_OFFLINE: "offline", + system.UF_OPAQUE: "opaque", + system.UF_READONLY: "rdonly", + system.UF_REPARSE: "reparse", + system.UF_SPARSE: "sparse", + system.UF_SYSTEM: "system", + } +) + +func parseFileFlags(fflags string) (uint32, uint32, error) { + var set, clear uint32 = 0, 0 + for _, fflag := range strings.Split(fflags, ",") { + isClear := false + if strings.HasPrefix(fflag, "no") { + isClear = true + fflag = strings.TrimPrefix(fflag, "no") + } + if value, ok := flagNameToValue[fflag]; ok { + if isClear { + clear |= value + } else { + set |= value + } + } else { + return 0, 0, fmt.Errorf("parsing file flags, unrecognised token: %s", fflag) + } + } + return set, clear, nil +} + +func formatFileFlags(fflags uint32) (string, error) { + var res = []string{} + for fflags != 0 { + // Extract lowest set bit + fflag := uint32(1) << bits.TrailingZeros32(fflags) + if name, ok := flagValueToName[fflag]; ok { + res = append(res, name) + } else { + return "", fmt.Errorf("formatting file flags, unrecognised flag: %x", fflag) + } + fflags &= ^fflag + } + return strings.Join(res, ","), nil +} + +func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error { + st, err := system.Lstat(path) + if err != nil { + return err + } + fflags, err := formatFileFlags(st.Flags()) + if err != nil { + return err + } + if fflags != "" { + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords[paxSCHILYFflags] = fflags + } + return nil +} + +func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error { + if fflags, ok := hdr.PAXRecords[paxSCHILYFflags]; ok { + var set, clear uint32 + set, clear, err := parseFileFlags(fflags) + if err != nil { + return err + } + + // Apply the delta to the existing file flags + st, err := system.Lstat(path) + if err != nil { + return err + } + return system.Lchflags(path, (st.Flags() & ^clear)|set) + } + return nil +} + +func resetImmutable(path string, fi *os.FileInfo) error { + var flags uint32 + if fi != nil { + flags = (*fi).Sys().(*syscall.Stat_t).Flags + } else { + st, err := system.Lstat(path) + if err != nil { + return err + } + flags = st.Flags() + } + if flags&(system.SF_IMMUTABLE|system.UF_IMMUTABLE) != 0 { + flags &= ^(system.SF_IMMUTABLE | system.UF_IMMUTABLE) + return system.Lchflags(path, flags) + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/archive/fflags_unsupported.go b/vendor/github.com/containers/storage/pkg/archive/fflags_unsupported.go new file mode 100644 index 00000000..27a0bae2 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/fflags_unsupported.go @@ -0,0 +1,21 @@ +//go:build !freebsd +// +build !freebsd + +package archive + +import ( + "archive/tar" + "os" +) + +func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error { + return nil +} + +func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error { + return nil +} + +func resetImmutable(path string, fi *os.FileInfo) error { + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/archive/time_unsupported.go b/vendor/github.com/containers/storage/pkg/archive/time_unsupported.go index e85aac05..8db64e80 100644 --- a/vendor/github.com/containers/storage/pkg/archive/time_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/archive/time_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package archive diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index b5d8961e..2de95f39 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -82,7 +82,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions } } - r := io.NopCloser(tarArchive) + r := tarArchive if decompress { decompressedArchive, err := archive.DecompressStream(tarArchive) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go index d257cc8e..42ee39f4 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go @@ -6,11 +6,7 @@ import ( "github.com/containers/storage/pkg/archive" ) -func chroot(path string) error { - return nil -} - -func invokeUnpack(decompressedArchive io.ReadCloser, +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { return archive.Unpack(decompressedArchive, dest, options) diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go index 8a5c680b..1395ff8c 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go @@ -12,7 +12,7 @@ func chroot(path string) error { return nil } -func invokeUnpack(decompressedArchive io.ReadCloser, +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { // Windows is different to Linux here because Windows does not support diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index d5aedd00..b03e9746 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -1,3 +1,4 @@ +//go:build !windows && !linux && !darwin // +build !windows,!linux,!darwin package chrootarchive diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go index 52c677bc..7b4ea9e5 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go @@ -27,13 +27,13 @@ func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions tmpDir, err := os.MkdirTemp(os.Getenv("temp"), "temp-storage-extract") if err != nil { - return 0, fmt.Errorf("ApplyLayer failed to create temp-storage-extract under %s. %s", dest, err) + return 0, fmt.Errorf("ApplyLayer failed to create temp-storage-extract under %s: %w", dest, err) } s, err := archive.UnpackLayer(dest, layer, options) os.RemoveAll(tmpDir) if err != nil { - return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %s", layer, dest, err) + return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %w", layer, dest, err) } return s, nil diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/jsoniter.go b/vendor/github.com/containers/storage/pkg/chrootarchive/jsoniter.go index 63f97045..40eec4dc 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/jsoniter.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/jsoniter.go @@ -1,3 +1,6 @@ +//go:build !windows && !darwin +// +build !windows,!darwin + package chrootarchive import jsoniter "github.com/json-iterator/go" diff --git a/vendor/github.com/containers/storage/pkg/chunked/compression.go b/vendor/github.com/containers/storage/pkg/chunked/compression.go index 8d4d3c4a..e828d479 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compression.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compression.go @@ -1,21 +1,10 @@ package chunked import ( - archivetar "archive/tar" - "bytes" - "encoding/binary" - "errors" - "fmt" "io" - "strconv" - "github.com/containerd/stargz-snapshotter/estargz" "github.com/containers/storage/pkg/chunked/compressor" "github.com/containers/storage/pkg/chunked/internal" - "github.com/klauspost/compress/zstd" - "github.com/klauspost/pgzip" - digest "github.com/opencontainers/go-digest" - "github.com/vbatts/tar-split/archive/tar" ) const ( @@ -29,247 +18,6 @@ const ( TypeSymlink = internal.TypeSymlink ) -var typesToTar = map[string]byte{ - TypeReg: tar.TypeReg, - TypeLink: tar.TypeLink, - TypeChar: tar.TypeChar, - TypeBlock: tar.TypeBlock, - TypeDir: tar.TypeDir, - TypeFifo: tar.TypeFifo, - TypeSymlink: tar.TypeSymlink, -} - -func typeToTarType(t string) (byte, error) { - r, found := typesToTar[t] - if !found { - return 0, fmt.Errorf("unknown type: %v", t) - } - return r, nil -} - -func isZstdChunkedFrameMagic(data []byte) bool { - if len(data) < 8 { - return false - } - return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8]) -} - -func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) { - // information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md - footerSize := int64(51) - if blobSize <= footerSize { - return nil, 0, errors.New("blob too small") - } - chunk := ImageSourceChunk{ - Offset: uint64(blobSize - footerSize), - Length: uint64(footerSize), - } - parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) - if err != nil { - return nil, 0, err - } - var reader io.ReadCloser - select { - case r := <-parts: - reader = r - case err := <-errs: - return nil, 0, err - } - defer reader.Close() - footer := make([]byte, footerSize) - if _, err := io.ReadFull(reader, footer); err != nil { - return nil, 0, err - } - - /* Read the ToC offset: - - 10 bytes gzip header - - 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ")) - - 2 bytes Extra: SI1 = 'S', SI2 = 'G' - - 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ")) - - 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC) - - 5 bytes flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0 - - 8 bytes gzip footer - */ - tocOffset, err := strconv.ParseInt(string(footer[16:16+22-6]), 16, 64) - if err != nil { - return nil, 0, fmt.Errorf("parse ToC offset: %w", err) - } - - size := int64(blobSize - footerSize - tocOffset) - // set a reasonable limit - if size > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") - } - - chunk = ImageSourceChunk{ - Offset: uint64(tocOffset), - Length: uint64(size), - } - parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk}) - if err != nil { - return nil, 0, err - } - - var tocReader io.ReadCloser - select { - case r := <-parts: - tocReader = r - case err := <-errs: - return nil, 0, err - } - defer tocReader.Close() - - r, err := pgzip.NewReader(tocReader) - if err != nil { - return nil, 0, err - } - defer r.Close() - - aTar := archivetar.NewReader(r) - - header, err := aTar.Next() - if err != nil { - return nil, 0, err - } - // set a reasonable limit - if header.Size > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") - } - - manifestUncompressed := make([]byte, header.Size) - if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil { - return nil, 0, err - } - - manifestDigester := digest.Canonical.Digester() - manifestChecksum := manifestDigester.Hash() - if _, err := manifestChecksum.Write(manifestUncompressed); err != nil { - return nil, 0, err - } - - d, err := digest.Parse(annotations[estargz.TOCJSONDigestAnnotation]) - if err != nil { - return nil, 0, err - } - if manifestDigester.Digest() != d { - return nil, 0, errors.New("invalid manifest checksum") - } - - return manifestUncompressed, tocOffset, nil -} - -// readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. The blob total size must -// be specified. -// This function uses the io.containers.zstd-chunked. annotations when specified. -func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) { - footerSize := int64(internal.FooterSizeSupported) - if blobSize <= footerSize { - return nil, 0, errors.New("blob too small") - } - - manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey] - if manifestChecksumAnnotation == "" { - return nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey) - } - - var offset, length, lengthUncompressed, manifestType uint64 - - if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" { - if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil { - return nil, 0, err - } - } else { - chunk := ImageSourceChunk{ - Offset: uint64(blobSize - footerSize), - Length: uint64(footerSize), - } - parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) - if err != nil { - return nil, 0, err - } - var reader io.ReadCloser - select { - case r := <-parts: - reader = r - case err := <-errs: - return nil, 0, err - } - footer := make([]byte, footerSize) - if _, err := io.ReadFull(reader, footer); err != nil { - return nil, 0, err - } - - offset = binary.LittleEndian.Uint64(footer[0:8]) - length = binary.LittleEndian.Uint64(footer[8:16]) - lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24]) - manifestType = binary.LittleEndian.Uint64(footer[24:32]) - if !isZstdChunkedFrameMagic(footer[32:40]) { - return nil, 0, errors.New("invalid magic number") - } - } - - if manifestType != internal.ManifestTypeCRFS { - return nil, 0, errors.New("invalid manifest type") - } - - // set a reasonable limit - if length > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") - } - if lengthUncompressed > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") - } - - chunk := ImageSourceChunk{ - Offset: offset, - Length: length, - } - - parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) - if err != nil { - return nil, 0, err - } - var reader io.ReadCloser - select { - case r := <-parts: - reader = r - case err := <-errs: - return nil, 0, err - } - - manifest := make([]byte, length) - if _, err := io.ReadFull(reader, manifest); err != nil { - return nil, 0, err - } - - manifestDigester := digest.Canonical.Digester() - manifestChecksum := manifestDigester.Hash() - if _, err := manifestChecksum.Write(manifest); err != nil { - return nil, 0, err - } - - d, err := digest.Parse(manifestChecksumAnnotation) - if err != nil { - return nil, 0, err - } - if manifestDigester.Digest() != d { - return nil, 0, errors.New("invalid manifest checksum") - } - - decoder, err := zstd.NewReader(nil) - if err != nil { - return nil, 0, err - } - defer decoder.Close() - - b := make([]byte, 0, lengthUncompressed) - if decoded, err := decoder.DecodeAll(manifest, b); err == nil { - return decoded, int64(offset), nil - } - - return manifest, int64(offset), nil -} - // ZstdCompressor is a CompressorFunc for the zstd compression algorithm. // Deprecated: Use pkg/chunked/compressor.ZstdCompressor. func ZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) { diff --git a/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go new file mode 100644 index 00000000..f6c7b77b --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go @@ -0,0 +1,259 @@ +package chunked + +import ( + archivetar "archive/tar" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "strconv" + + "github.com/containerd/stargz-snapshotter/estargz" + "github.com/containers/storage/pkg/chunked/internal" + "github.com/klauspost/compress/zstd" + "github.com/klauspost/pgzip" + digest "github.com/opencontainers/go-digest" + "github.com/vbatts/tar-split/archive/tar" +) + +var typesToTar = map[string]byte{ + TypeReg: tar.TypeReg, + TypeLink: tar.TypeLink, + TypeChar: tar.TypeChar, + TypeBlock: tar.TypeBlock, + TypeDir: tar.TypeDir, + TypeFifo: tar.TypeFifo, + TypeSymlink: tar.TypeSymlink, +} + +func typeToTarType(t string) (byte, error) { + r, found := typesToTar[t] + if !found { + return 0, fmt.Errorf("unknown type: %v", t) + } + return r, nil +} + +func isZstdChunkedFrameMagic(data []byte) bool { + if len(data) < 8 { + return false + } + return bytes.Equal(internal.ZstdChunkedFrameMagic, data[:8]) +} + +func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) { + // information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md + footerSize := int64(51) + if blobSize <= footerSize { + return nil, 0, errors.New("blob too small") + } + chunk := ImageSourceChunk{ + Offset: uint64(blobSize - footerSize), + Length: uint64(footerSize), + } + parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, 0, err + } + var reader io.ReadCloser + select { + case r := <-parts: + reader = r + case err := <-errs: + return nil, 0, err + } + defer reader.Close() + footer := make([]byte, footerSize) + if _, err := io.ReadFull(reader, footer); err != nil { + return nil, 0, err + } + + /* Read the ToC offset: + - 10 bytes gzip header + - 2 bytes XLEN (length of Extra field) = 26 (4 bytes header + 16 hex digits + len("STARGZ")) + - 2 bytes Extra: SI1 = 'S', SI2 = 'G' + - 2 bytes Extra: LEN = 22 (16 hex digits + len("STARGZ")) + - 22 bytes Extra: subfield = fmt.Sprintf("%016xSTARGZ", offsetOfTOC) + - 5 bytes flate header: BFINAL = 1(last block), BTYPE = 0(non-compressed block), LEN = 0 + - 8 bytes gzip footer + */ + tocOffset, err := strconv.ParseInt(string(footer[16:16+22-6]), 16, 64) + if err != nil { + return nil, 0, fmt.Errorf("parse ToC offset: %w", err) + } + + size := int64(blobSize - footerSize - tocOffset) + // set a reasonable limit + if size > (1<<20)*50 { + return nil, 0, errors.New("manifest too big") + } + + chunk = ImageSourceChunk{ + Offset: uint64(tocOffset), + Length: uint64(size), + } + parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, 0, err + } + + var tocReader io.ReadCloser + select { + case r := <-parts: + tocReader = r + case err := <-errs: + return nil, 0, err + } + defer tocReader.Close() + + r, err := pgzip.NewReader(tocReader) + if err != nil { + return nil, 0, err + } + defer r.Close() + + aTar := archivetar.NewReader(r) + + header, err := aTar.Next() + if err != nil { + return nil, 0, err + } + // set a reasonable limit + if header.Size > (1<<20)*50 { + return nil, 0, errors.New("manifest too big") + } + + manifestUncompressed := make([]byte, header.Size) + if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil { + return nil, 0, err + } + + manifestDigester := digest.Canonical.Digester() + manifestChecksum := manifestDigester.Hash() + if _, err := manifestChecksum.Write(manifestUncompressed); err != nil { + return nil, 0, err + } + + d, err := digest.Parse(annotations[estargz.TOCJSONDigestAnnotation]) + if err != nil { + return nil, 0, err + } + if manifestDigester.Digest() != d { + return nil, 0, errors.New("invalid manifest checksum") + } + + return manifestUncompressed, tocOffset, nil +} + +// readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. The blob total size must +// be specified. +// This function uses the io.github.containers.zstd-chunked. annotations when specified. +func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, int64, error) { + footerSize := int64(internal.FooterSizeSupported) + if blobSize <= footerSize { + return nil, 0, errors.New("blob too small") + } + + manifestChecksumAnnotation := annotations[internal.ManifestChecksumKey] + if manifestChecksumAnnotation == "" { + return nil, 0, fmt.Errorf("manifest checksum annotation %q not found", internal.ManifestChecksumKey) + } + + var offset, length, lengthUncompressed, manifestType uint64 + + if offsetMetadata := annotations[internal.ManifestInfoKey]; offsetMetadata != "" { + if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil { + return nil, 0, err + } + } else { + chunk := ImageSourceChunk{ + Offset: uint64(blobSize - footerSize), + Length: uint64(footerSize), + } + parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, 0, err + } + var reader io.ReadCloser + select { + case r := <-parts: + reader = r + case err := <-errs: + return nil, 0, err + } + footer := make([]byte, footerSize) + if _, err := io.ReadFull(reader, footer); err != nil { + return nil, 0, err + } + + offset = binary.LittleEndian.Uint64(footer[0:8]) + length = binary.LittleEndian.Uint64(footer[8:16]) + lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24]) + manifestType = binary.LittleEndian.Uint64(footer[24:32]) + if !isZstdChunkedFrameMagic(footer[32:40]) { + return nil, 0, errors.New("invalid magic number") + } + } + + if manifestType != internal.ManifestTypeCRFS { + return nil, 0, errors.New("invalid manifest type") + } + + // set a reasonable limit + if length > (1<<20)*50 { + return nil, 0, errors.New("manifest too big") + } + if lengthUncompressed > (1<<20)*50 { + return nil, 0, errors.New("manifest too big") + } + + chunk := ImageSourceChunk{ + Offset: offset, + Length: length, + } + + parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + if err != nil { + return nil, 0, err + } + var reader io.ReadCloser + select { + case r := <-parts: + reader = r + case err := <-errs: + return nil, 0, err + } + + manifest := make([]byte, length) + if _, err := io.ReadFull(reader, manifest); err != nil { + return nil, 0, err + } + + manifestDigester := digest.Canonical.Digester() + manifestChecksum := manifestDigester.Hash() + if _, err := manifestChecksum.Write(manifest); err != nil { + return nil, 0, err + } + + d, err := digest.Parse(manifestChecksumAnnotation) + if err != nil { + return nil, 0, err + } + if manifestDigester.Digest() != d { + return nil, 0, errors.New("invalid manifest checksum") + } + + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, 0, err + } + defer decoder.Close() + + b := make([]byte, 0, lengthUncompressed) + if decoded, err := decoder.DecodeAll(manifest, b); err == nil { + return decoded, int64(offset), nil + } + + return manifest, int64(offset), nil +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go index 362c168d..2a9bdc67 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go @@ -33,11 +33,11 @@ const ( holesFinderStateEOF ) -// ReadByte reads a single byte from the underlying reader. +// readByte reads a single byte from the underlying reader. // If a single byte is read, the return value is (0, RAW-BYTE-VALUE, nil). // If there are at least f.THRESHOLD consecutive zeros, then the // return value is (N_CONSECUTIVE_ZEROS, '\x00'). -func (f *holesFinder) ReadByte() (int64, byte, error) { +func (f *holesFinder) readByte() (int64, byte, error) { for { switch f.state { // reading the file stream @@ -78,7 +78,7 @@ func (f *holesFinder) ReadByte() (int64, byte, error) { f.state = holesFinderStateFound } } else { - if f.reader.UnreadByte(); err != nil { + if err := f.reader.UnreadByte(); err != nil { return 0, 0, err } f.state = holesFinderStateRead @@ -95,7 +95,7 @@ func (f *holesFinder) ReadByte() (int64, byte, error) { return holeLen, 0, nil } if b != 0 { - if f.reader.UnreadByte(); err != nil { + if err := f.reader.UnreadByte(); err != nil { return 0, 0, err } f.state = holesFinderStateRead @@ -159,7 +159,7 @@ func (rc *rollingChecksumReader) Read(b []byte) (bool, int, error) { } for i := 0; i < len(b); i++ { - holeLen, n, err := rc.reader.ReadByte() + holeLen, n, err := rc.reader.readByte() if err != nil { if err == io.EOF { rc.closed = true @@ -429,7 +429,7 @@ func zstdChunkedWriterWithLevel(out io.Writer, metadata map[string]string, level go func() { ch <- writeZstdChunkedStream(out, metadata, r, level) - io.Copy(io.Discard, r) + _, _ = io.Copy(io.Discard, r) // Ordinarily writeZstdChunkedStream consumes all of r. If it fails, ensure the write end never blocks and eventually terminates. r.Close() close(ch) }() diff --git a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go b/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go index 3bb5286d..092b0353 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go +++ b/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go @@ -88,8 +88,8 @@ func GetType(t byte) (string, error) { } const ( - ManifestChecksumKey = "io.containers.zstd-chunked.manifest-checksum" - ManifestInfoKey = "io.containers.zstd-chunked.manifest-position" + ManifestChecksumKey = "io.github.containers.zstd-chunked.manifest-checksum" + ManifestInfoKey = "io.github.containers.zstd-chunked.manifest-position" // ManifestTypeCRFS is a manifest file compatible with the CRFS TOC file. ManifestTypeCRFS = 1 @@ -114,7 +114,7 @@ func appendZstdSkippableFrame(dest io.Writer, data []byte) error { return err } - var size []byte = make([]byte, 4) + size := make([]byte, 4) binary.LittleEndian.PutUint32(size, uint32(len(data))) if _, err := dest.Write(size); err != nil { return err @@ -168,7 +168,7 @@ func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, off } // Store the offset to the manifest and its size in LE order - var manifestDataLE []byte = make([]byte, FooterSizeSupported) + manifestDataLE := make([]byte, FooterSizeSupported) binary.LittleEndian.PutUint64(manifestDataLE, manifestOffset) binary.LittleEndian.PutUint64(manifestDataLE[8:], uint64(len(compressedManifest))) binary.LittleEndian.PutUint64(manifestDataLE[16:], uint64(len(manifest))) diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage.go b/vendor/github.com/containers/storage/pkg/chunked/storage.go index f0bd3627..752ee252 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage.go @@ -17,7 +17,7 @@ type ImageSourceSeekable interface { } // ErrBadRequest is returned when the request is not valid -type ErrBadRequest struct { +type ErrBadRequest struct { //nolint: errname } func (e ErrBadRequest) Error() string { diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_log.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_log.go index 082fb1ba..ead15c14 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_log.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_log.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go index 190d83d4..7baca812 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go index 7f793c27..071f7f35 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go @@ -1,3 +1,4 @@ +//go:build linux && cgo && !libdm_no_deferred_remove // +build linux,cgo,!libdm_no_deferred_remove package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_dynamic.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_dynamic.go index 7d845089..93dcc322 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_dynamic.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_dynamic.go @@ -1,3 +1,4 @@ +//go:build linux && cgo && !static_build // +build linux,cgo,!static_build package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go index a880fec8..91906f2e 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -1,3 +1,4 @@ +//go:build linux && cgo && libdm_no_deferred_remove // +build linux,cgo,libdm_no_deferred_remove package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_static.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_static.go index cf7f26a4..68ea48fe 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_static.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_static.go @@ -1,3 +1,4 @@ +//go:build linux && cgo && static_build // +build linux,cgo,static_build package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/ioctl.go b/vendor/github.com/containers/storage/pkg/devicemapper/ioctl.go index 50ea7c48..90ffe2c3 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/ioctl.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/ioctl.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package devicemapper diff --git a/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go b/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go index 7df7f3d4..e883d25f 100644 --- a/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go +++ b/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package dmesg diff --git a/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go b/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go index abf6e2f8..e3cef48a 100644 --- a/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go +++ b/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go @@ -321,14 +321,14 @@ func ReadSymlinkedDirectory(path string) (string, error) { var realPath string var err error if realPath, err = filepath.Abs(path); err != nil { - return "", fmt.Errorf("unable to get absolute path for %s: %s", path, err) + return "", fmt.Errorf("unable to get absolute path for %s: %w", path, err) } if realPath, err = filepath.EvalSymlinks(realPath); err != nil { - return "", fmt.Errorf("failed to canonicalise path for %s: %s", path, err) + return "", fmt.Errorf("failed to canonicalise path for %s: %w", path, err) } realPathInfo, err := os.Stat(realPath) if err != nil { - return "", fmt.Errorf("failed to stat target '%s' of '%s': %s", realPath, path, err) + return "", fmt.Errorf("failed to stat target '%s' of '%s': %w", realPath, path, err) } if !realPathInfo.Mode().IsDir() { return "", fmt.Errorf("canonical path points to a file '%s'", realPath) diff --git a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go b/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go index 027db259..0883ee02 100644 --- a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go +++ b/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go @@ -1,3 +1,4 @@ +//go:build !linux && !darwin && !freebsd // +build !linux,!darwin,!freebsd package homedir diff --git a/vendor/github.com/containers/storage/pkg/homedir/homedir_unix.go b/vendor/github.com/containers/storage/pkg/homedir/homedir_unix.go index 37dc9159..9976f19a 100644 --- a/vendor/github.com/containers/storage/pkg/homedir/homedir_unix.go +++ b/vendor/github.com/containers/storage/pkg/homedir/homedir_unix.go @@ -63,7 +63,7 @@ func StickRuntimeDirContents(files []string) ([]string, error) { runtimeDir, err := GetRuntimeDir() if err != nil { // ignore error if runtimeDir is empty - return nil, nil + return nil, nil //nolint: nilerr } runtimeDir, err = filepath.Abs(runtimeDir) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index a5760906..4cec8b26 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -2,6 +2,7 @@ package idtools import ( "bufio" + "errors" "fmt" "os" "os/user" @@ -359,7 +360,8 @@ func parseSubidFile(path, username string) (ranges, error) { } func checkChownErr(err error, name string, uid, gid int) error { - if e, ok := err.(*os.PathError); ok && e.Err == syscall.EINVAL { + var e *os.PathError + if errors.As(err, &e) && e.Err == syscall.EINVAL { return fmt.Errorf("potentially insufficient UIDs or GIDs available in user namespace (requested %d:%d for %s): Check /etc/subuid and /etc/subgid if configured locally and run podman-system-migrate: %w", uid, gid, name, err) } return err diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go index 84da1b76..78141fb8 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux || !libsubid || !cgo // +build !linux !libsubid !cgo package idtools diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_windows.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_windows.go index 16be94f4..dc69c607 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools_windows.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package idtools diff --git a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go index a74893e8..17eed4e6 100644 --- a/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go +++ b/vendor/github.com/containers/storage/pkg/ioutils/fswriters.go @@ -15,7 +15,7 @@ type AtomicFileWriterOptions struct { NoSync bool } -var defaultWriterOptions AtomicFileWriterOptions = AtomicFileWriterOptions{} +var defaultWriterOptions = AtomicFileWriterOptions{} // SetDefaultOptions overrides the default options used when creating an // atomic file writer. @@ -27,6 +27,13 @@ func SetDefaultOptions(opts AtomicFileWriterOptions) { // temporary file and closing it atomically changes the temporary file to // destination path. Writing and closing concurrently is not allowed. func NewAtomicFileWriterWithOpts(filename string, perm os.FileMode, opts *AtomicFileWriterOptions) (io.WriteCloser, error) { + return newAtomicFileWriter(filename, perm, opts) +} + +// newAtomicFileWriter returns WriteCloser so that writing to it writes to a +// temporary file and closing it atomically changes the temporary file to +// destination path. Writing and closing concurrently is not allowed. +func newAtomicFileWriter(filename string, perm os.FileMode, opts *AtomicFileWriterOptions) (*atomicFileWriter, error) { f, err := os.CreateTemp(filepath.Dir(filename), ".tmp-"+filepath.Base(filename)) if err != nil { return nil, err @@ -55,14 +62,14 @@ func NewAtomicFileWriter(filename string, perm os.FileMode) (io.WriteCloser, err // AtomicWriteFile atomically writes data to a file named by filename. func AtomicWriteFile(filename string, data []byte, perm os.FileMode) error { - f, err := NewAtomicFileWriter(filename, perm) + f, err := newAtomicFileWriter(filename, perm, nil) if err != nil { return err } n, err := f.Write(data) if err == nil && n < len(data) { err = io.ErrShortWrite - f.(*atomicFileWriter).writeErr = err + f.writeErr = err } if err1 := f.Close(); err == nil { err = err1 diff --git a/vendor/github.com/containers/storage/pkg/ioutils/fswriters_unsupported.go b/vendor/github.com/containers/storage/pkg/ioutils/fswriters_unsupported.go index 79a09403..63548928 100644 --- a/vendor/github.com/containers/storage/pkg/ioutils/fswriters_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/ioutils/fswriters_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package ioutils diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go index d3f4df09..f639357f 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile.go @@ -17,10 +17,6 @@ type Locker interface { // - tried to lock a read-only lock-file Lock() - // Acquire a writer lock recursively, allowing for recursive acquisitions - // within the same process space. - RecursiveLock() - // Unlock the lock. // The default unix implementation panics if: // - unlocking an unlocked lock @@ -44,8 +40,13 @@ type Locker interface { // IsReadWrite() checks if the lock file is read-write IsReadWrite() bool - // Locked() checks if lock is locked for writing by a thread in this process - Locked() bool + // AssertLocked() can be used by callers that _know_ that they hold the lock (for reading or writing), for sanity checking. + // It might do nothing at all, or it may panic if the caller is not the owner of this lock. + AssertLocked() + + // AssertLocked() can be used by callers that _know_ that they hold the lock locked for writing, for sanity checking. + // It might do nothing at all, or it may panic if the caller is not the owner of this lock for writing. + AssertLockedForWriting() } var ( diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go index 3c242016..21378e94 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go @@ -30,7 +30,6 @@ type lockfile struct { locktype int16 locked bool ro bool - recursive bool } const lastWriterIDSize = 64 // This must be the same as len(stringid.GenerateRandomID) @@ -79,7 +78,7 @@ func openLock(path string, ro bool) (fd int, err error) { } fd, err = unix.Open(path, flags, 0o644) if err == nil { - return + return fd, nil } // the directory of the lockfile seems to be removed, try to create it @@ -131,10 +130,10 @@ func createLockerForPath(path string, ro bool) (Locker, error) { // lock locks the lockfile via FCTNL(2) based on the specified type and // command. -func (l *lockfile) lock(lType int16, recursive bool) { +func (l *lockfile) lock(lType int16) { lk := unix.Flock_t{ Type: lType, - Whence: int16(os.SEEK_SET), + Whence: int16(unix.SEEK_SET), Start: 0, Len: 0, } @@ -142,13 +141,7 @@ func (l *lockfile) lock(lType int16, recursive bool) { case unix.F_RDLCK: l.rwMutex.RLock() case unix.F_WRLCK: - if recursive { - // NOTE: that's okay as recursive is only set in RecursiveLock(), so - // there's no need to protect against hypothetical RDLCK cases. - l.rwMutex.RLock() - } else { - l.rwMutex.Lock() - } + l.rwMutex.Lock() default: panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", lType)) } @@ -171,7 +164,6 @@ func (l *lockfile) lock(lType int16, recursive bool) { } l.locktype = lType l.locked = true - l.recursive = recursive l.counter++ } @@ -180,30 +172,19 @@ func (l *lockfile) Lock() { if l.ro { panic("can't take write lock on read-only lock file") } else { - l.lock(unix.F_WRLCK, false) - } -} - -// RecursiveLock locks the lockfile as a writer but allows for recursive -// acquisitions within the same process space. Note that RLock() will be called -// if it's a lockTypReader lock. -func (l *lockfile) RecursiveLock() { - if l.ro { - l.RLock() - } else { - l.lock(unix.F_WRLCK, true) + l.lock(unix.F_WRLCK) } } // LockRead locks the lockfile as a reader. func (l *lockfile) RLock() { - l.lock(unix.F_RDLCK, false) + l.lock(unix.F_RDLCK) } // Unlock unlocks the lockfile. func (l *lockfile) Unlock() { l.stateMutex.Lock() - if l.locked == false { + if !l.locked { // Panic when unlocking an unlocked lock. That's a violation // of the lock semantics and will reveal such. panic("calling Unlock on unlocked lock") @@ -224,7 +205,7 @@ func (l *lockfile) Unlock() { // file lock. unix.Close(int(l.fd)) } - if l.locktype == unix.F_RDLCK || l.recursive { + if l.locktype == unix.F_RDLCK { l.rwMutex.RUnlock() } else { l.rwMutex.Unlock() @@ -232,11 +213,33 @@ func (l *lockfile) Unlock() { l.stateMutex.Unlock() } -// Locked checks if lockfile is locked for writing by a thread in this process. -func (l *lockfile) Locked() bool { - l.stateMutex.Lock() - defer l.stateMutex.Unlock() - return l.locked && (l.locktype == unix.F_WRLCK) +func (l *lockfile) AssertLocked() { + // DO NOT provide a variant that returns the value of l.locked. + // + // If the caller does not hold the lock, l.locked might nevertheless be true because another goroutine does hold it, and + // we can’t tell the difference. + // + // Hence, this “AssertLocked” method, which exists only for sanity checks. + + // Don’t even bother with l.stateMutex: The caller is expected to hold the lock, and in that case l.locked is constant true + // with no possible writers. + // If the caller does not hold the lock, we are violating the locking/memory model anyway, and accessing the data + // without the lock is more efficient for callers, and potentially more visible to lock analysers for incorrect callers. + if !l.locked { + panic("internal error: lock is not held by the expected owner") + } +} + +func (l *lockfile) AssertLockedForWriting() { + // DO NOT provide a variant that returns the current lock state. + // + // The same caveats as for AssertLocked apply equally. + + l.AssertLocked() + // Like AssertLocked, don’t even bother with l.stateMutex. + if l.locktype != unix.F_WRLCK { + panic("internal error: lock is not held for writing") + } } // Touch updates the lock file with the UID of the user. @@ -265,14 +268,15 @@ func (l *lockfile) Modified() (bool, error) { panic("attempted to check last-writer in lockfile without locking it first") } defer l.stateMutex.Unlock() - currentLW := make([]byte, len(l.lw)) + currentLW := make([]byte, lastWriterIDSize) n, err := unix.Pread(int(l.fd), currentLW, 0) if err != nil { return true, err } - if n != len(l.lw) { - return true, nil - } + // It is important to handle the partial read case, because + // the initial size of the lock file is zero, which is a valid + // state (no writes yet) + currentLW = currentLW[:n] oldLW := l.lw l.lw = currentLW return !bytes.Equal(currentLW, oldLW), nil diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_windows.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_windows.go index 82bd91db..cc898d82 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_windows.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package lockfile @@ -36,12 +37,6 @@ func (l *lockfile) Lock() { l.locked = true } -func (l *lockfile) RecursiveLock() { - // We don't support Windows but a recursive writer-lock in one process-space - // is really a writer lock, so just panic. - panic("not supported") -} - func (l *lockfile) RLock() { l.mu.Lock() l.locked = true @@ -52,8 +47,23 @@ func (l *lockfile) Unlock() { l.mu.Unlock() } -func (l *lockfile) Locked() bool { - return l.locked +func (l *lockfile) AssertLocked() { + // DO NOT provide a variant that returns the value of l.locked. + // + // If the caller does not hold the lock, l.locked might nevertheless be true because another goroutine does hold it, and + // we can’t tell the difference. + // + // Hence, this “AssertLocked” method, which exists only for sanity checks. + if !l.locked { + panic("internal error: lock is not held by the expected owner") + } +} + +func (l *lockfile) AssertLockedForWriting() { + // DO NOT provide a variant that returns the current lock state. + // + // The same caveats as for AssertLocked apply equally. + l.AssertLocked() // The current implementation does not distinguish between read and write locks. } func (l *lockfile) Modified() (bool, error) { diff --git a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go index 6f072650..de10e332 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go index ea684195..da2ba46f 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go +++ b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go index a50de7f0..21a98100 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loopback.go b/vendor/github.com/containers/storage/pkg/loopback/loopback.go index c9be0577..ec428724 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loopback.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/mount/unmount_unix.go b/vendor/github.com/containers/storage/pkg/mount/unmount_unix.go index 1d1afeee..a2a1d407 100644 --- a/vendor/github.com/containers/storage/pkg/mount/unmount_unix.go +++ b/vendor/github.com/containers/storage/pkg/mount/unmount_unix.go @@ -1,16 +1,29 @@ +//go:build !windows // +build !windows package mount -import "golang.org/x/sys/unix" +import ( + "time" + + "golang.org/x/sys/unix" +) func unmount(target string, flags int) error { - err := unix.Unmount(target, flags) - if err == nil || err == unix.EINVAL { - // Ignore "not mounted" error here. Note the same error - // can be returned if flags are invalid, so this code - // assumes that the flags value is always correct. - return nil + var err error + for i := 0; i < 50; i++ { + err = unix.Unmount(target, flags) + switch err { + case unix.EBUSY: + time.Sleep(50 * time.Millisecond) + continue + case unix.EINVAL, nil: + // Ignore "not mounted" error here. Note the same error + // can be returned if flags are invalid, so this code + // assumes that the flags value is always correct. + return nil + } + break } return &mountError{ diff --git a/vendor/github.com/containers/storage/pkg/mount/unmount_unsupported.go b/vendor/github.com/containers/storage/pkg/mount/unmount_unsupported.go index eebc4ab8..d3a0cf51 100644 --- a/vendor/github.com/containers/storage/pkg/mount/unmount_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/mount/unmount_unsupported.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package mount diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel.go index 7738fc74..6406cb14 100644 --- a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel.go +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows // Package kernel provides helper function to get, parse and compare kernel diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_darwin.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_darwin.go index 20d67f78..645790da 100644 --- a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_darwin.go +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_darwin.go @@ -43,7 +43,7 @@ func getRelease() (string, error) { prettyNames, err := shellwords.Parse(content[1]) if err != nil { - return "", fmt.Errorf("kernel version is invalid: %s", err.Error()) + return "", fmt.Errorf("kernel version is invalid: %w", err) } if len(prettyNames) != 2 { diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_unix.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_unix.go index 7a68bc39..ed8cca2c 100644 --- a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_unix.go +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_unix.go @@ -1,3 +1,4 @@ +//go:build linux || freebsd || solaris || openbsd // +build linux freebsd solaris openbsd // Package kernel provides helper function to get, parse and compare kernel diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_windows.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_windows.go index 3d382923..b30da9fa 100644 --- a/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_windows.go +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package kernel diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported.go index 1da3f239..052c6874 100644 --- a/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported.go @@ -1,4 +1,5 @@ -// +build !linux,!solaris +//go:build freebsd || openbsd +// +build freebsd openbsd package kernel @@ -6,13 +7,7 @@ import ( "errors" ) -// Utsname represents the system name structure. -// It is defined here to make it portable as it is available on linux but not -// on windows. -type Utsname struct { - Release [65]byte -} - +// A stub called by kernel_unix.go . func uname() (*Utsname, error) { return nil, errors.New("Kernel version detection is available only on linux") } diff --git a/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported_type.go b/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported_type.go new file mode 100644 index 00000000..b7e0f0c2 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported_type.go @@ -0,0 +1,11 @@ +//go:build !linux && !solaris +// +build !linux,!solaris + +package kernel + +// Utsname represents the system name structure. +// It is defined here to make it portable as it is available on linux but not +// on windows. +type Utsname struct { + Release [65]byte +} diff --git a/vendor/github.com/containers/storage/pkg/reexec/command_freebsd.go b/vendor/github.com/containers/storage/pkg/reexec/command_freebsd.go index 6f63ae99..32d6a9f4 100644 --- a/vendor/github.com/containers/storage/pkg/reexec/command_freebsd.go +++ b/vendor/github.com/containers/storage/pkg/reexec/command_freebsd.go @@ -1,3 +1,4 @@ +//go:build freebsd // +build freebsd package reexec diff --git a/vendor/github.com/containers/storage/pkg/reexec/command_linux.go b/vendor/github.com/containers/storage/pkg/reexec/command_linux.go index d3dd86d3..87b43ed9 100644 --- a/vendor/github.com/containers/storage/pkg/reexec/command_linux.go +++ b/vendor/github.com/containers/storage/pkg/reexec/command_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package reexec diff --git a/vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go b/vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go index 5b3605f3..77c93b4a 100644 --- a/vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux && !windows && !freebsd && !solaris && !darwin // +build !linux,!windows,!freebsd,!solaris,!darwin package reexec diff --git a/vendor/github.com/containers/storage/pkg/reexec/command_windows.go b/vendor/github.com/containers/storage/pkg/reexec/command_windows.go index d8685647..c46125eb 100644 --- a/vendor/github.com/containers/storage/pkg/reexec/command_windows.go +++ b/vendor/github.com/containers/storage/pkg/reexec/command_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package reexec diff --git a/vendor/github.com/containers/storage/pkg/system/chtimes_unix.go b/vendor/github.com/containers/storage/pkg/system/chtimes_unix.go index 09d58bcb..a208a6b5 100644 --- a/vendor/github.com/containers/storage/pkg/system/chtimes_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/chtimes_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/chtimes_windows.go b/vendor/github.com/containers/storage/pkg/system/chtimes_windows.go index 45428c14..bfb621dc 100644 --- a/vendor/github.com/containers/storage/pkg/system/chtimes_windows.go +++ b/vendor/github.com/containers/storage/pkg/system/chtimes_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/init.go b/vendor/github.com/containers/storage/pkg/system/init.go index 17935088..05642f60 100644 --- a/vendor/github.com/containers/storage/pkg/system/init.go +++ b/vendor/github.com/containers/storage/pkg/system/init.go @@ -6,7 +6,7 @@ import ( "unsafe" ) -// Used by chtimes +// maxTime is used by chtimes. var maxTime time.Time func init() { diff --git a/vendor/github.com/containers/storage/pkg/system/lchflags_bsd.go b/vendor/github.com/containers/storage/pkg/system/lchflags_bsd.go new file mode 100644 index 00000000..4eaeb5d6 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/lchflags_bsd.go @@ -0,0 +1,56 @@ +//go:build freebsd +// +build freebsd + +package system + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +// Flag values from +const ( + /* + * Definitions of flags stored in file flags word. + * + * Super-user and owner changeable flags. + */ + UF_SETTABLE uint32 = 0x0000ffff /* mask of owner changeable flags */ + UF_NODUMP uint32 = 0x00000001 /* do not dump file */ + UF_IMMUTABLE uint32 = 0x00000002 /* file may not be changed */ + UF_APPEND uint32 = 0x00000004 /* writes to file may only append */ + UF_OPAQUE uint32 = 0x00000008 /* directory is opaque wrt. union */ + UF_NOUNLINK uint32 = 0x00000010 /* file may not be removed or renamed */ + + UF_SYSTEM uint32 = 0x00000080 /* Windows system file bit */ + UF_SPARSE uint32 = 0x00000100 /* sparse file */ + UF_OFFLINE uint32 = 0x00000200 /* file is offline */ + UF_REPARSE uint32 = 0x00000400 /* Windows reparse point file bit */ + UF_ARCHIVE uint32 = 0x00000800 /* file needs to be archived */ + UF_READONLY uint32 = 0x00001000 /* Windows readonly file bit */ + /* This is the same as the MacOS X definition of UF_HIDDEN. */ + UF_HIDDEN uint32 = 0x00008000 /* file is hidden */ + + /* + * Super-user changeable flags. + */ + SF_SETTABLE uint32 = 0xffff0000 /* mask of superuser changeable flags */ + SF_ARCHIVED uint32 = 0x00010000 /* file is archived */ + SF_IMMUTABLE uint32 = 0x00020000 /* file may not be changed */ + SF_APPEND uint32 = 0x00040000 /* writes to file may only append */ + SF_NOUNLINK uint32 = 0x00100000 /* file may not be removed or renamed */ + SF_SNAPSHOT uint32 = 0x00200000 /* snapshot inode */ +) + +func Lchflags(path string, flags uint32) error { + p, err := unix.BytePtrFromString(path) + if err != nil { + return err + } + _, _, e1 := unix.Syscall(unix.SYS_LCHFLAGS, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) + if e1 != 0 { + return e1 + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/system/lcow_unix.go b/vendor/github.com/containers/storage/pkg/system/lcow_unix.go index cff33bb4..42658c8b 100644 --- a/vendor/github.com/containers/storage/pkg/system/lcow_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/lcow_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/lstat_unix.go b/vendor/github.com/containers/storage/pkg/system/lstat_unix.go index e9d301f0..9b13e614 100644 --- a/vendor/github.com/containers/storage/pkg/system/lstat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/lstat_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system @@ -14,7 +15,7 @@ import ( func Lstat(path string) (*StatT, error) { s := &syscall.Stat_t{} if err := syscall.Lstat(path, s); err != nil { - return nil, &os.PathError{"Lstat", path, err} + return nil, &os.PathError{Op: "Lstat", Path: path, Err: err} } return fromStatT(s) } diff --git a/vendor/github.com/containers/storage/pkg/system/mknod_windows.go b/vendor/github.com/containers/storage/pkg/system/mknod_windows.go index 2e863c02..c35b1b34 100644 --- a/vendor/github.com/containers/storage/pkg/system/mknod_windows.go +++ b/vendor/github.com/containers/storage/pkg/system/mknod_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/path_unix.go b/vendor/github.com/containers/storage/pkg/system/path_unix.go index f3762e69..ff01143e 100644 --- a/vendor/github.com/containers/storage/pkg/system/path_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/path_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/process_unix.go b/vendor/github.com/containers/storage/pkg/system/process_unix.go index a9a0dd75..7ee59d92 100644 --- a/vendor/github.com/containers/storage/pkg/system/process_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/process_unix.go @@ -1,3 +1,4 @@ +//go:build linux || freebsd || solaris || darwin // +build linux freebsd solaris darwin package system diff --git a/vendor/github.com/containers/storage/pkg/system/rm.go b/vendor/github.com/containers/storage/pkg/system/rm.go index 5d63dc74..60c7d8bd 100644 --- a/vendor/github.com/containers/storage/pkg/system/rm.go +++ b/vendor/github.com/containers/storage/pkg/system/rm.go @@ -1,6 +1,7 @@ package system import ( + "errors" "fmt" "os" "syscall" @@ -35,14 +36,24 @@ func EnsureRemoveAll(dir string) error { } for { - if err := resetFileFlags(dir); err != nil { - return fmt.Errorf("resetting file flags: %w", err) - } err := os.RemoveAll(dir) if err == nil { return nil } + // If the RemoveAll fails with a permission error, we + // may have immutable files so try to remove the + // immutable flag and redo the RemoveAll. + if errors.Is(err, syscall.EPERM) { + if err = resetFileFlags(dir); err != nil { + return fmt.Errorf("resetting file flags: %w", err) + } + err = os.RemoveAll(dir) + if err == nil { + return nil + } + } + pe, ok := err.(*os.PathError) if !ok { return err @@ -65,7 +76,7 @@ func EnsureRemoveAll(dir string) error { continue } - if pe.Err != syscall.EBUSY { + if !IsEBUSY(pe.Err) { return err } diff --git a/vendor/github.com/containers/storage/pkg/system/rm_freebsd.go b/vendor/github.com/containers/storage/pkg/system/rm_freebsd.go index 35896c11..39a5de7a 100644 --- a/vendor/github.com/containers/storage/pkg/system/rm_freebsd.go +++ b/vendor/github.com/containers/storage/pkg/system/rm_freebsd.go @@ -3,28 +3,13 @@ package system import ( "io/fs" "path/filepath" - "unsafe" - - "golang.org/x/sys/unix" ) -func lchflags(path string, flags int) (err error) { - p, err := unix.BytePtrFromString(path) - if err != nil { - return err - } - _, _, e1 := unix.Syscall(unix.SYS_LCHFLAGS, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) - if e1 != 0 { - return e1 - } - return nil -} - // Reset file flags in a directory tree. This allows EnsureRemoveAll // to delete trees which have the immutable flag set. func resetFileFlags(dir string) error { return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err := lchflags(path, 0); err != nil { + if err := Lchflags(path, 0); err != nil { return err } return nil diff --git a/vendor/github.com/containers/storage/pkg/system/stat_common.go b/vendor/github.com/containers/storage/pkg/system/stat_common.go new file mode 100644 index 00000000..e965c54c --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/stat_common.go @@ -0,0 +1,13 @@ +//go:build !freebsd +// +build !freebsd + +package system + +type platformStatT struct { +} + +// Flags return file flags if supported or zero otherwise +func (s StatT) Flags() uint32 { + _ = s.platformStatT // Silence warnings that StatT.platformStatT is unused (on these platforms) + return 0 +} diff --git a/vendor/github.com/containers/storage/pkg/system/stat_freebsd.go b/vendor/github.com/containers/storage/pkg/system/stat_freebsd.go index 715f05b9..2c70bb03 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_freebsd.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_freebsd.go @@ -2,12 +2,23 @@ package system import "syscall" +type platformStatT struct { + flags uint32 +} + +// Flags return file flags if supported or zero otherwise +func (s StatT) Flags() uint32 { + return s.flags +} + // fromStatT converts a syscall.Stat_t type to a system.Stat_t type func fromStatT(s *syscall.Stat_t) (*StatT, error) { - return &StatT{size: s.Size, + st := &StatT{size: s.Size, mode: uint32(s.Mode), uid: s.Uid, gid: s.Gid, rdev: uint64(s.Rdev), - mtim: s.Mtimespec}, nil + mtim: s.Mtimespec} + st.flags = s.Flags + return st, nil } diff --git a/vendor/github.com/containers/storage/pkg/system/stat_unix.go b/vendor/github.com/containers/storage/pkg/system/stat_unix.go index 2fac918b..23c4c780 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system @@ -17,6 +18,7 @@ type StatT struct { rdev uint64 size int64 mtim syscall.Timespec + platformStatT } // Mode returns file's permission mode. diff --git a/vendor/github.com/containers/storage/pkg/system/stat_windows.go b/vendor/github.com/containers/storage/pkg/system/stat_windows.go index d3063605..9c4d28d4 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_windows.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_windows.go @@ -11,6 +11,7 @@ type StatT struct { mode os.FileMode size int64 mtim time.Time + platformStatT } // Size returns file's size. diff --git a/vendor/github.com/containers/storage/pkg/system/umask.go b/vendor/github.com/containers/storage/pkg/system/umask.go index 5a10eda5..ad0337db 100644 --- a/vendor/github.com/containers/storage/pkg/system/umask.go +++ b/vendor/github.com/containers/storage/pkg/system/umask.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/umask_windows.go b/vendor/github.com/containers/storage/pkg/system/umask_windows.go index 13f1de17..9497596a 100644 --- a/vendor/github.com/containers/storage/pkg/system/umask_windows.go +++ b/vendor/github.com/containers/storage/pkg/system/umask_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package system diff --git a/vendor/github.com/containers/storage/pkg/system/utimes_unsupported.go b/vendor/github.com/containers/storage/pkg/system/utimes_unsupported.go index 13971454..843ecdc5 100644 --- a/vendor/github.com/containers/storage/pkg/system/utimes_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/system/utimes_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux && !freebsd // +build !linux,!freebsd package system diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go index 221eb78b..8bd7acf1 100644 --- a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go @@ -1,3 +1,4 @@ +//go:build !linux && !darwin // +build !linux,!darwin package system diff --git a/vendor/github.com/containers/storage/pkg/truncindex/truncindex.go b/vendor/github.com/containers/storage/pkg/truncindex/truncindex.go index 74776e65..8f073265 100644 --- a/vendor/github.com/containers/storage/pkg/truncindex/truncindex.go +++ b/vendor/github.com/containers/storage/pkg/truncindex/truncindex.go @@ -14,7 +14,7 @@ import ( var ( // ErrEmptyPrefix is an error returned if the prefix was empty. - ErrEmptyPrefix = errors.New("Prefix can't be empty") + ErrEmptyPrefix = errors.New("prefix can't be empty") // ErrIllegalChar is returned when a space is in the ID ErrIllegalChar = errors.New("illegal character: ' '") @@ -25,7 +25,7 @@ var ( // ErrAmbiguousPrefix is returned if the prefix was ambiguous // (multiple ids for the prefix). -type ErrAmbiguousPrefix struct { +type ErrAmbiguousPrefix struct { //nolint: errname prefix string } @@ -42,6 +42,7 @@ type TruncIndex struct { } // NewTruncIndex creates a new TruncIndex and initializes with a list of IDs. +// Invalid IDs are _silently_ ignored. func NewTruncIndex(ids []string) (idx *TruncIndex) { idx = &TruncIndex{ ids: make(map[string]struct{}), @@ -51,7 +52,7 @@ func NewTruncIndex(ids []string) (idx *TruncIndex) { trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)), } for _, id := range ids { - idx.addID(id) + _ = idx.addID(id) // Ignore invalid IDs. Duplicate IDs are not a problem. } return } @@ -132,7 +133,8 @@ func (idx *TruncIndex) Get(s string) (string, error) { func (idx *TruncIndex) Iterate(handler func(id string)) { idx.Lock() defer idx.Unlock() - idx.trie.Visit(func(prefix patricia.Prefix, item patricia.Item) error { + // Ignore the error from Visit: it can only fail if the provided visitor fails, and ours never does. + _ = idx.trie.Visit(func(prefix patricia.Prefix, item patricia.Item) error { handler(string(prefix)) return nil }) diff --git a/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_cgo.go b/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_cgo.go index 4f441c32..08dbc661 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_cgo.go +++ b/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_cgo.go @@ -1,3 +1,4 @@ +//go:build linux && cgo // +build linux,cgo package unshare diff --git a/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_nocgo.go b/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_nocgo.go index a5005403..25054810 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_nocgo.go +++ b/vendor/github.com/containers/storage/pkg/unshare/getenv_linux_nocgo.go @@ -1,3 +1,4 @@ +//go:build linux && !cgo // +build linux,!cgo package unshare diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare.go b/vendor/github.com/containers/storage/pkg/unshare/unshare.go index c854fdf5..00f397f3 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare.go @@ -5,18 +5,12 @@ import ( "os" "os/user" "sync" - - "github.com/sirupsen/logrus" ) var ( homeDirOnce sync.Once homeDirErr error homeDir string - - hasCapSysAdminOnce sync.Once - hasCapSysAdminRet bool - hasCapSysAdminErr error ) // HomeDir returns the home directory for the current user. @@ -36,14 +30,3 @@ func HomeDir() (string, error) { }) return homeDir, homeDirErr } - -func bailOnError(err error, format string, a ...interface{}) { // nolint: golint,goprintffuncname - if err != nil { - if format != "" { - logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err) - } else { - logrus.Errorf("%v", err) - } - os.Exit(1) - } -} diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go index 01cf33bd..86ac12ec 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go @@ -1,3 +1,4 @@ +//go:build darwin // +build darwin package unshare diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_gccgo.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_gccgo.go index 39bff651..21a43d38 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_gccgo.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_gccgo.go @@ -1,3 +1,4 @@ +//go:build linux && cgo && gccgo // +build linux,cgo,gccgo package unshare diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go index 3fc36201..86922846 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go @@ -400,6 +400,12 @@ func hasFullUsersMappings() (bool, error) { return bytes.Contains(content, []byte("4294967295")), nil } +var ( + hasCapSysAdminOnce sync.Once + hasCapSysAdminRet bool + hasCapSysAdminErr error +) + // IsRootless tells us if we are running in rootless mode func IsRootless() bool { isRootlessOnce.Do(func() { @@ -445,10 +451,21 @@ type Runnable interface { Run() error } +func bailOnError(err error, format string, a ...interface{}) { // nolint: golint,goprintffuncname + if err != nil { + if format != "" { + logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err) + } else { + logrus.Errorf("%v", err) + } + os.Exit(1) + } +} + // MaybeReexecUsingUserNamespace re-exec the process in a new namespace func MaybeReexecUsingUserNamespace(evenForRoot bool) { // If we've already been through this once, no need to try again. - if os.Geteuid() == 0 && IsRootless() { + if os.Geteuid() == 0 && GetRootlessUID() > 0 { return } diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index fb1faaa1..ad3ca273 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/go-multierror" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" ) type updateNameOperation int @@ -49,57 +50,27 @@ var ( storesLock sync.Mutex ) -// ROFileBasedStore wraps up the methods of the various types of file-based -// data stores that we implement which are needed for both read-only and -// read-write files. -type ROFileBasedStore interface { - Locker - - // Load reloads the contents of the store from disk. It should be called - // with the lock held. - Load() error - - // ReloadIfChanged reloads the contents of the store from disk if it is changed. - ReloadIfChanged() error -} - -// RWFileBasedStore wraps up the methods of various types of file-based data -// stores that we implement using read-write files. -type RWFileBasedStore interface { - // Save saves the contents of the store to disk. It should be called with - // the lock held, and Touch() should be called afterward before releasing the - // lock. - Save() error -} - -// FileBasedStore wraps up the common methods of various types of file-based -// data stores that we implement. -type FileBasedStore interface { - ROFileBasedStore - RWFileBasedStore -} - -// ROMetadataStore wraps a method for reading metadata associated with an ID. -type ROMetadataStore interface { +// roMetadataStore wraps a method for reading metadata associated with an ID. +type roMetadataStore interface { // Metadata reads metadata associated with an item with the specified ID. Metadata(id string) (string, error) } -// RWMetadataStore wraps a method for setting metadata associated with an ID. -type RWMetadataStore interface { +// rwMetadataStore wraps a method for setting metadata associated with an ID. +type rwMetadataStore interface { // SetMetadata updates the metadata associated with the item with the specified ID. SetMetadata(id, metadata string) error } -// MetadataStore wraps up methods for getting and setting metadata associated with IDs. -type MetadataStore interface { - ROMetadataStore - RWMetadataStore +// metadataStore wraps up methods for getting and setting metadata associated with IDs. +type metadataStore interface { + roMetadataStore + rwMetadataStore } -// An ROBigDataStore wraps up the read-only big-data related methods of the +// An roBigDataStore wraps up the read-only big-data related methods of the // various types of file-based lookaside stores that we implement. -type ROBigDataStore interface { +type roBigDataStore interface { // BigData retrieves a (potentially large) piece of data associated with // this ID, if it has previously been set. BigData(id, key string) ([]byte, error) @@ -117,8 +88,8 @@ type ROBigDataStore interface { BigDataNames(id string) ([]string, error) } -// A RWImageBigDataStore wraps up how we store big-data associated with images. -type RWImageBigDataStore interface { +// A rwImageBigDataStore wraps up how we store big-data associated with images. +type rwImageBigDataStore interface { // SetBigData stores a (potentially large) piece of data associated // with this ID. // Pass github.com/containers/image/manifest.Digest as digestManifest @@ -126,16 +97,16 @@ type RWImageBigDataStore interface { SetBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error } -// A ContainerBigDataStore wraps up how we store big-data associated with containers. -type ContainerBigDataStore interface { - ROBigDataStore +// A containerBigDataStore wraps up how we store big-data associated with containers. +type containerBigDataStore interface { + roBigDataStore // SetBigData stores a (potentially large) piece of data associated // with this ID. SetBigData(id, key string, data []byte) error } -// A ROLayerBigDataStore wraps up how we store RO big-data associated with layers. -type ROLayerBigDataStore interface { +// A roLayerBigDataStore wraps up how we store RO big-data associated with layers. +type roLayerBigDataStore interface { // SetBigData stores a (potentially large) piece of data associated // with this ID. BigData(id, key string) (io.ReadCloser, error) @@ -145,21 +116,15 @@ type ROLayerBigDataStore interface { BigDataNames(id string) ([]string, error) } -// A RWLayerBigDataStore wraps up how we store big-data associated with layers. -type RWLayerBigDataStore interface { +// A rwLayerBigDataStore wraps up how we store big-data associated with layers. +type rwLayerBigDataStore interface { // SetBigData stores a (potentially large) piece of data associated // with this ID. SetBigData(id, key string, data io.Reader) error } -// A LayerBigDataStore wraps up how we store big-data associated with layers. -type LayerBigDataStore interface { - ROLayerBigDataStore - RWLayerBigDataStore -} - -// A FlaggableStore can have flags set and cleared on items which it manages. -type FlaggableStore interface { +// A flaggableStore can have flags set and cleared on items which it manages. +type flaggableStore interface { // ClearFlag removes a named flag from an item in the store. ClearFlag(id string, flag string) error @@ -622,11 +587,11 @@ type store struct { autoNsMinSize uint32 autoNsMaxSize uint32 graphDriver drivers.Driver - layerStore LayerStore - roLayerStores []ROLayerStore - imageStore ImageStore - roImageStores []ROImageStore - containerStore ContainerStore + layerStore rwLayerStore + roLayerStores []roLayerStore + imageStore rwImageStore + roImageStores []roImageStore + containerStore rwContainerStore digestLockRoot string disableVolatile bool } @@ -699,10 +664,8 @@ func GetStore(options types.StoreOptions) (Store, error) { if err := os.MkdirAll(options.GraphRoot, 0700); err != nil { return nil, err } - for _, subdir := range []string{"mounts", "tmp", options.GraphDriverName} { - if err := os.MkdirAll(filepath.Join(options.GraphRoot, subdir), 0700); err != nil { - return nil, err - } + if err := os.MkdirAll(filepath.Join(options.GraphRoot, options.GraphDriverName), 0700); err != nil { + return nil, err } graphLock, err := GetLockfile(filepath.Join(options.GraphRoot, "storage.lock")) @@ -823,7 +786,7 @@ func (s *store) load() error { return err } s.imageStore = ris - if _, err := s.ROImageStores(); err != nil { + if _, err := s.getROImageStores(); err != nil { return err } @@ -894,10 +857,9 @@ func (s *store) GraphDriver() (drivers.Driver, error) { return s.getGraphDriver() } -// LayerStore obtains and returns a handle to the writeable layer store object -// used by the Store. Accessing this store directly will bypass locking and -// synchronization, so it is not a part of the exported Store interface. -func (s *store) LayerStore() (LayerStore, error) { +// getLayerStore obtains and returns a handle to the writeable layer store object +// used by the Store. +func (s *store) getLayerStore() (rwLayerStore, error) { s.graphLock.Lock() defer s.graphLock.Unlock() if s.graphLock.TouchedSince(s.lastLoaded) { @@ -929,10 +891,9 @@ func (s *store) LayerStore() (LayerStore, error) { return s.layerStore, nil } -// ROLayerStores obtains additional read/only layer store objects used by the -// Store. Accessing these stores directly will bypass locking and -// synchronization, so it is not part of the exported Store interface. -func (s *store) ROLayerStores() ([]ROLayerStore, error) { +// getROLayerStores obtains additional read/only layer store objects used by the +// Store. +func (s *store) getROLayerStores() ([]roLayerStore, error) { s.graphLock.Lock() defer s.graphLock.Unlock() if s.roLayerStores != nil { @@ -958,20 +919,85 @@ func (s *store) ROLayerStores() ([]ROLayerStore, error) { return s.roLayerStores, nil } -// ImageStore obtains and returns a handle to the writable image store object -// used by the Store. Accessing this store directly will bypass locking and -// synchronization, so it is not a part of the exported Store interface. -func (s *store) ImageStore() (ImageStore, error) { +// allLayerStores returns a list of all layer store objects used by the Store. +// This is a convenience method for read-only users of the Store. +func (s *store) allLayerStores() ([]roLayerStore, error) { + primary, err := s.getLayerStore() + if err != nil { + return nil, fmt.Errorf("loading primary layer store data: %w", err) + } + additional, err := s.getROLayerStores() + if err != nil { + return nil, fmt.Errorf("loading additional layer stores: %w", err) + } + return append([]roLayerStore{primary}, additional...), nil +} + +// readAllLayerStores processes allLayerStores() in order: +// It locks the store for reading, checks for updates, and calls +// +// (done, err) := fn(store) +// +// until the callback returns done == true, and returns the data from the callback. +// +// If reading any layer store fails, it immediately returns (true, err). +// +// If all layer stores are processed without setting done == true, it returns (false, nil). +// +// Typical usage: +// +// var res T = failureValue +// if done, err := s.readAllLayerStores(store, func(…) { +// … +// }; done { +// return res, err +// } +func (s *store) readAllLayerStores(fn func(store roLayerStore) (bool, error)) (bool, error) { + layerStores, err := s.allLayerStores() + if err != nil { + return true, err + } + for _, s := range layerStores { + store := s + if err := store.startReading(); err != nil { + return true, err + } + defer store.stopReading() + if done, err := fn(store); done { + return true, err + } + } + return false, nil +} + +// writeToLayerStore is a helper for working with store.getLayerStore(): +// It locks the store for writing, checks for updates, and calls fn() +// It returns the return value of fn, or its own error initializing the store. +func (s *store) writeToLayerStore(fn func(store rwLayerStore) error) error { + store, err := s.getLayerStore() + if err != nil { + return err + } + + if err := store.startWriting(); err != nil { + return err + } + defer store.stopWriting() + return fn(store) +} + +// getImageStore obtains and returns a handle to the writable image store object +// used by the Store. +func (s *store) getImageStore() (rwImageStore, error) { if s.imageStore != nil { return s.imageStore, nil } return nil, ErrLoadError } -// ROImageStores obtains additional read/only image store objects used by the -// Store. Accessing these stores directly will bypass locking and -// synchronization, so it is not a part of the exported Store interface. -func (s *store) ROImageStores() ([]ROImageStore, error) { +// getROImageStores obtains additional read/only image store objects used by the +// Store. +func (s *store) getROImageStores() ([]roImageStore, error) { if s.imageStore == nil { return nil, ErrLoadError } @@ -979,16 +1005,131 @@ func (s *store) ROImageStores() ([]ROImageStore, error) { return s.roImageStores, nil } -// ContainerStore obtains and returns a handle to the container store object -// used by the Store. Accessing this store directly will bypass locking and -// synchronization, so it is not a part of the exported Store interface. -func (s *store) ContainerStore() (ContainerStore, error) { +// allImageStores returns a list of all image store objects used by the Store. +// This is a convenience method for read-only users of the Store. +func (s *store) allImageStores() ([]roImageStore, error) { + primary, err := s.getImageStore() + if err != nil { + return nil, fmt.Errorf("loading primary image store data: %w", err) + } + additional, err := s.getROImageStores() + if err != nil { + return nil, fmt.Errorf("loading additional image stores: %w", err) + } + return append([]roImageStore{primary}, additional...), nil +} + +// readAllImageStores processes allImageStores() in order: +// It locks the store for reading, checks for updates, and calls +// +// (done, err) := fn(store) +// +// until the callback returns done == true, and returns the data from the callback. +// +// If reading any Image store fails, it immediately returns (true, err). +// +// If all Image stores are processed without setting done == true, it returns (false, nil). +// +// Typical usage: +// +// var res T = failureValue +// if done, err := s.readAllImageStores(store, func(…) { +// … +// }; done { +// return res, err +// } +func (s *store) readAllImageStores(fn func(store roImageStore) (bool, error)) (bool, error) { + ImageStores, err := s.allImageStores() + if err != nil { + return true, err + } + for _, s := range ImageStores { + store := s + if err := store.startReading(); err != nil { + return true, err + } + defer store.stopReading() + if done, err := fn(store); done { + return true, err + } + } + return false, nil +} + +// writeToImageStore is a convenience helper for working with store.getImageStore(): +// It locks the store for writing, checks for updates, and calls fn() +// It returns the return value of fn, or its own error initializing the store. +func (s *store) writeToImageStore(fn func(store rwImageStore) error) error { + store, err := s.getImageStore() + if err != nil { + return err + } + + if err := store.startWriting(); err != nil { + return err + } + defer store.stopWriting() + return fn(store) +} + +// getContainerStore obtains and returns a handle to the container store object +// used by the Store. +func (s *store) getContainerStore() (rwContainerStore, error) { if s.containerStore != nil { return s.containerStore, nil } return nil, ErrLoadError } +// writeToContainerStore is a convenience helper for working with store.getContainerStore(): +// It locks the store for writing, checks for updates, and calls fn() +// It returns the return value of fn, or its own error initializing the store. +func (s *store) writeToContainerStore(fn func(store rwContainerStore) error) error { + store, err := s.getContainerStore() + if err != nil { + return err + } + + if err := store.startWriting(); err != nil { + return err + } + defer store.stopWriting() + return fn(store) +} + +// writeToAllStores is a convenience helper for writing to all three stores: +// It locks the stores for writing, checks for updates, and calls fn(). +// It returns the return value of fn, or its own error initializing the stores. +func (s *store) writeToAllStores(fn func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error) error { + rlstore, err := s.getLayerStore() + if err != nil { + return err + } + ristore, err := s.getImageStore() + if err != nil { + return err + } + rcstore, err := s.getContainerStore() + if err != nil { + return err + } + + if err := rlstore.startWriting(); err != nil { + return err + } + defer rlstore.stopWriting() + if err := ristore.startWriting(); err != nil { + return err + } + defer ristore.stopWriting() + if err := rcstore.startWriting(); err != nil { + return err + } + defer rcstore.stopWriting() + + return fn(rlstore, ristore, rcstore) +} + func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { if s.graphDriver == nil || !s.graphDriver.SupportsShifting() { return false @@ -1004,28 +1145,26 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions, diff io.Reader) (*Layer, int64, error) { var parentLayer *Layer - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return nil, -1, err } - rlstores, err := s.ROLayerStores() + rlstores, err := s.getROLayerStores() if err != nil { return nil, -1, err } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, -1, err } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return nil, -1, err } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + defer rlstore.stopWriting() + if err := rcstore.startWriting(); err != nil { return nil, -1, err } + defer rcstore.stopWriting() if options == nil { options = &LayerOptions{} } @@ -1039,14 +1178,13 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w gidMap := options.GIDMap if parent != "" { var ilayer *Layer - for _, l := range append([]ROLayerStore{rlstore}, rlstores...) { + for _, l := range append([]roLayerStore{rlstore}, rlstores...) { lstore := l if lstore != rlstore { - lstore.RLock() - defer lstore.Unlock() - if err := lstore.ReloadIfChanged(); err != nil { + if err := lstore.startReading(); err != nil { return nil, -1, err } + defer lstore.stopReading() } if l, err := lstore.Get(parent); err == nil && l != nil { ilayer = l @@ -1105,27 +1243,17 @@ func (s *store) CreateLayer(id, parent string, names []string, mountLabel string func (s *store) CreateImage(id string, names []string, layer, metadata string, options *ImageOptions) (*Image, error) { if layer != "" { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() + layerStores, err := s.allLayerStores() if err != nil { return nil, err } var ilayer *Layer - for _, s := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range layerStores { store := s - if store == lstore { - store.Lock() - } else { - store.RLock() - } - defer store.Unlock() - err := store.ReloadIfChanged() - if err != nil { + if err := store.startReading(); err != nil { return nil, err } + defer store.stopReading() ilayer, err = store.Get(layer) if err == nil { break @@ -1137,25 +1265,26 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o layer = ilayer.ID } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return nil, err - } + var res *Image + err := s.writeToImageStore(func(ristore rwImageStore) error { + creationDate := time.Now().UTC() + if options != nil && !options.CreationDate.IsZero() { + creationDate = options.CreationDate + } - creationDate := time.Now().UTC() - if options != nil && !options.CreationDate.IsZero() { - creationDate = options.CreationDate - } - - return ristore.Create(id, names, layer, metadata, creationDate, options.Digest) + var err error + res, err = ristore.Create(id, names, layer, metadata, creationDate, options.Digest) + return err + }) + return res, err } -func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, createMappedLayer bool, rlstore LayerStore, lstores []ROLayerStore, options types.IDMappingOptions) (*Layer, error) { +// imageTopLayerForMapping does ??? +// On entry: +// - ristore must be locked EITHER for reading or writing +// - rlstore must be locked for writing +// - lstores must all be locked for reading +func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, createMappedLayer bool, rlstore rwLayerStore, lstores []roLayerStore, options types.IDMappingOptions) (*Layer, error) { layerMatchesMappingOptions := func(layer *Layer, options types.IDMappingOptions) bool { // If the driver supports shifting and the layer has no mappings, we can use it. if s.canUseShifting(options.UIDMap, options.GIDMap) && len(layer.UIDMap) == 0 && len(layer.GIDMap) == 0 { @@ -1172,17 +1301,10 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, crea return reflect.DeepEqual(layer.UIDMap, options.UIDMap) && reflect.DeepEqual(layer.GIDMap, options.GIDMap) } var layer, parentLayer *Layer - allStores := append([]ROLayerStore{rlstore}, lstores...) + allStores := append([]roLayerStore{rlstore}, lstores...) // Locate the image's top layer and its parent, if it has one. for _, s := range allStores { store := s - if store != rlstore { - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } - } // Walk the top layer list. for _, candidate := range append([]string{image.TopLayer}, image.MappedTopLayers...) { if cLayer, err := store.Get(candidate); err == nil { @@ -1278,7 +1400,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat if options.HostGIDMapping { options.GIDMap = nil } - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return nil, err } @@ -1295,45 +1417,55 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat defer s.usernsLock.Unlock() } - var imageHomeStore ROImageStore - var istore ImageStore - var istores []ROImageStore - var lstores []ROLayerStore - var cimage *Image + var imageHomeStore roImageStore // Set if image != "" + var istore rwImageStore // Set, and locked read-write, if image != "" + var istores []roImageStore // Set, and NOT NECESSARILY ALL locked read-only, if image != "" + var lstores []roLayerStore // Set, and locked read-only, if image != "" + var cimage *Image // Set if image != "" if image != "" { var err error - lstores, err = s.ROLayerStores() + lstores, err = s.getROLayerStores() if err != nil { return nil, err } - istore, err = s.ImageStore() + istore, err = s.getImageStore() if err != nil { return nil, err } - istores, err = s.ROImageStores() + istores, err = s.getROImageStores() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return nil, err } - for _, s := range append([]ROImageStore{istore}, istores...) { + defer rlstore.stopWriting() + for _, s := range lstores { store := s - if store == istore { - store.Lock() - } else { - store.RLock() - } - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { + if err := store.startReading(); err != nil { return nil, err } - cimage, err = store.Get(image) - if err == nil { - imageHomeStore = store - break + defer store.stopReading() + } + if err := istore.startWriting(); err != nil { + return nil, err + } + defer istore.stopWriting() + cimage, err = istore.Get(image) + if err == nil { + imageHomeStore = istore + } else { + for _, s := range istores { + store := s + if err := store.startReading(); err != nil { + return nil, err + } + defer store.stopReading() + cimage, err = store.Get(image) + if err == nil { + imageHomeStore = store + break + } } } if cimage == nil { @@ -1344,7 +1476,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat if options.AutoUserNs { var err error - options.UIDMap, options.GIDMap, err = s.getAutoUserNS(&options.AutoUserNsOpts, cimage) + options.UIDMap, options.GIDMap, err = s.getAutoUserNS(&options.AutoUserNsOpts, cimage, rlstore, lstores) if err != nil { return nil, err } @@ -1371,11 +1503,10 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } } } else { - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return nil, err } + defer rlstore.stopWriting() if !options.HostUIDMapping && len(options.UIDMap) == 0 { uidMap = s.uidMap } @@ -1427,120 +1558,79 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat return nil, err } layer = clayer.ID - rcstore, err := s.ContainerStore() - if err != nil { - return nil, err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return nil, err - } - options.IDMappingOptions = types.IDMappingOptions{ - HostUIDMapping: len(options.UIDMap) == 0, - HostGIDMapping: len(options.GIDMap) == 0, - UIDMap: copyIDMap(options.UIDMap), - GIDMap: copyIDMap(options.GIDMap), - } - container, err := rcstore.Create(id, names, imageID, layer, metadata, options) - if err != nil || container == nil { - rlstore.Delete(layer) - } + + var container *Container + err = s.writeToContainerStore(func(rcstore rwContainerStore) error { + options.IDMappingOptions = types.IDMappingOptions{ + HostUIDMapping: len(options.UIDMap) == 0, + HostGIDMapping: len(options.GIDMap) == 0, + UIDMap: copyIDMap(options.UIDMap), + GIDMap: copyIDMap(options.GIDMap), + } + var err error + container, err = rcstore.Create(id, names, imageID, layer, metadata, options) + if err != nil || container == nil { + if err2 := rlstore.Delete(layer); err2 != nil { + if err == nil { + err = fmt.Errorf("deleting layer %#v: %w", layer, err2) + } else { + logrus.Errorf("While recovering from a failure to create a container, error deleting layer %#v: %v", layer, err2) + } + } + } + return err + }) return container, err } func (s *store) SetMetadata(id, metadata string) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - - if rlstore.Exists(id) { - return rlstore.SetMetadata(id, metadata) - } - if ristore.Exists(id) { - return ristore.SetMetadata(id, metadata) - } - if rcstore.Exists(id) { - return rcstore.SetMetadata(id, metadata) - } - return ErrNotAnID + return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if rlstore.Exists(id) { + return rlstore.SetMetadata(id, metadata) + } + if ristore.Exists(id) { + return ristore.SetMetadata(id, metadata) + } + if rcstore.Exists(id) { + return rcstore.SetMetadata(id, metadata) + } + return ErrNotAnID + }) } func (s *store) Metadata(id string) (string, error) { - lstore, err := s.LayerStore() - if err != nil { - return "", err - } - lstores, err := s.ROLayerStores() - if err != nil { - return "", err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return "", err - } + var res string + + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if store.Exists(id) { - return store.Metadata(id) + var err error + res, err = store.Metadata(id) + return true, err } + return false, nil + }); done { + return res, err } - istore, err := s.ImageStore() - if err != nil { - return "", err - } - istores, err := s.ROImageStores() - if err != nil { - return "", err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return "", err - } + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { if store.Exists(id) { - return store.Metadata(id) + var err error + res, err = store.Metadata(id) + return true, err } + return false, nil + }); done { + return res, err } - cstore, err := s.ContainerStore() + cstore, err := s.getContainerStore() if err != nil { return "", err } - cstore.RLock() - defer cstore.Unlock() - if err := cstore.ReloadIfChanged(); err != nil { + if err := cstore.startReading(); err != nil { return "", err } + defer cstore.stopReading() if cstore.Exists(id) { return cstore.Metadata(id) } @@ -1548,102 +1638,65 @@ func (s *store) Metadata(id string) (string, error) { } func (s *store) ListImageBigData(id string) ([]string, error) { - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res []string + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { bigDataNames, err := store.BigDataNames(id) if err == nil { - return bigDataNames, err + res = bigDataNames + return true, nil } + return false, nil + }); done { + return res, err } return nil, fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown) } func (s *store) ImageBigDataSize(id, key string) (int64, error) { - istore, err := s.ImageStore() - if err != nil { - return -1, err - } - istores, err := s.ROImageStores() - if err != nil { - return -1, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return -1, err - } + var res int64 = -1 + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { size, err := store.BigDataSize(id, key) if err == nil { - return size, nil + res = size + return true, nil } + return false, nil + }); done { + return res, err } return -1, ErrSizeUnknown } func (s *store) ImageBigDataDigest(id, key string) (digest.Digest, error) { - ristore, err := s.ImageStore() - if err != nil { - return "", err - } - stores, err := s.ROImageStores() - if err != nil { - return "", err - } - stores = append([]ROImageStore{ristore}, stores...) - for _, r := range stores { - ristore := r - ristore.RLock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return "", err - } + var res digest.Digest + if done, err := s.readAllImageStores(func(ristore roImageStore) (bool, error) { d, err := ristore.BigDataDigest(id, key) if err == nil && d.Validate() == nil { - return d, nil + res = d + return true, nil } + return false, nil + }); done { + return res, err } return "", ErrDigestUnknown } func (s *store) ImageBigData(id, key string) ([]byte, error) { - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } foundImage := false - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res []byte + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { data, err := store.BigData(id, key) if err == nil { - return data, nil + res = data + return true, nil } if store.Exists(id) { foundImage = true } + return false, nil + }); done { + return res, err } if foundImage { return nil, fmt.Errorf("locating item named %q for image with ID %q (consider removing the image to resolve the issue): %w", key, id, os.ErrNotExist) @@ -1654,29 +1707,20 @@ func (s *store) ImageBigData(id, key string) ([]byte, error) { // ListLayerBigData retrieves a list of the (possibly large) chunks of // named data associated with an layer. func (s *store) ListLayerBigData(id string) ([]string, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } foundLayer := false - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res []string + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { data, err := store.BigDataNames(id) if err == nil { - return data, nil + res = data + return true, nil } if store.Exists(id) { foundLayer = true } + return false, nil + }); done { + return res, err } if foundLayer { return nil, fmt.Errorf("locating big data for layer with ID %q: %w", id, os.ErrNotExist) @@ -1687,29 +1731,20 @@ func (s *store) ListLayerBigData(id string) ([]string, error) { // LayerBigData retrieves a (possibly large) chunk of named data // associated with a layer. func (s *store) LayerBigData(id, key string) (io.ReadCloser, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } foundLayer := false - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res io.ReadCloser + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { data, err := store.BigData(id, key) if err == nil { - return data, nil + res = data + return true, nil } if store.Exists(id) { foundLayer = true } + return false, nil + }); done { + return res, err } if foundLayer { return nil, fmt.Errorf("locating item named %q for layer with ID %q: %w", key, id, os.ErrNotExist) @@ -1720,72 +1755,43 @@ func (s *store) LayerBigData(id, key string) (io.ReadCloser, error) { // SetLayerBigData stores a (possibly large) chunk of named data // associated with a layer. func (s *store) SetLayerBigData(id, key string, data io.Reader) error { - store, err := s.LayerStore() - if err != nil { - return err - } - - store.Lock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return err - } - return store.SetBigData(id, key, data) + return s.writeToLayerStore(func(store rwLayerStore) error { + return store.SetBigData(id, key, data) + }) } func (s *store) SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error { - ristore, err := s.ImageStore() - if err != nil { - return err - } - - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - - return ristore.SetBigData(id, key, data, digestManifest) + return s.writeToImageStore(func(ristore rwImageStore) error { + return ristore.SetBigData(id, key, data, digestManifest) + }) } func (s *store) ImageSize(id string) (int64, error) { - var image *Image - - lstore, err := s.LayerStore() + layerStores, err := s.allLayerStores() if err != nil { - return -1, fmt.Errorf("loading primary layer store data: %w", err) + return -1, err } - lstores, err := s.ROLayerStores() - if err != nil { - return -1, fmt.Errorf("loading additional layer stores: %w", err) - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range layerStores { store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { + if err := store.startReading(); err != nil { return -1, err } + defer store.stopReading() } - var imageStore ROBigDataStore - istore, err := s.ImageStore() + imageStores, err := s.allImageStores() if err != nil { - return -1, fmt.Errorf("loading primary image store data: %w", err) + return -1, err } - istores, err := s.ROImageStores() - if err != nil { - return -1, fmt.Errorf("loading additional image stores: %w", err) - } - // Look for the image's record. - for _, s := range append([]ROImageStore{istore}, istores...) { + var imageStore roBigDataStore + var image *Image + for _, s := range imageStores { store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { + if err := store.startReading(); err != nil { return -1, err } + defer store.stopReading() if image, err = store.Get(id); err == nil { imageStore = store break @@ -1813,9 +1819,9 @@ func (s *store) ImageSize(id string) (int64, error) { } visited[layerID] = struct{}{} // Look for the layer and the store that knows about it. - var layerStore ROLayerStore + var layerStore roLayerStore var layer *Layer - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, store := range layerStores { if layer, err = store.Get(layerID); err == nil { layerStore = store break @@ -1859,21 +1865,16 @@ func (s *store) ImageSize(id string) (int64, error) { } func (s *store) ContainerSize(id string) (int64, error) { - lstore, err := s.LayerStore() + layerStores, err := s.allLayerStores() if err != nil { return -1, err } - lstores, err := s.ROLayerStores() - if err != nil { - return -1, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range layerStores { store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { + if err := store.startReading(); err != nil { return -1, err } + defer store.stopReading() } // Get the location of the container directory and container run directory. @@ -1887,15 +1888,14 @@ func (s *store) ContainerSize(id string) (int64, error) { return -1, err } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return -1, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return -1, err } + defer rcstore.stopReading() // Read the container record. container, err := rcstore.Get(id) @@ -1906,7 +1906,7 @@ func (s *store) ContainerSize(id string) (int64, error) { // Read the container's layer's size. var layer *Layer var size int64 - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, store := range layerStores { if layer, err = store.Get(container.LayerID); err == nil { size, err = store.DiffSize("", layer.ID) if err != nil { @@ -1948,127 +1948,93 @@ func (s *store) ContainerSize(id string) (int64, error) { } func (s *store) ListContainerBigData(id string) ([]string, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() return rcstore.BigDataNames(id) } func (s *store) ContainerBigDataSize(id, key string) (int64, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return -1, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return -1, err } + defer rcstore.stopReading() return rcstore.BigDataSize(id, key) } func (s *store) ContainerBigDataDigest(id, key string) (digest.Digest, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return "", err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return "", err } + defer rcstore.stopReading() return rcstore.BigDataDigest(id, key) } func (s *store) ContainerBigData(id, key string) ([]byte, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() return rcstore.BigData(id, key) } func (s *store) SetContainerBigData(id, key string, data []byte) error { - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - return rcstore.SetBigData(id, key, data) + return s.writeToContainerStore(func(rcstore rwContainerStore) error { + return rcstore.SetBigData(id, key, data) + }) } func (s *store) Exists(id string) bool { - lstore, err := s.LayerStore() - if err != nil { - return false - } - lstores, err := s.ROLayerStores() - if err != nil { - return false - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return false - } + var res = false + + if done, _ := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if store.Exists(id) { - return true + res = true + return true, nil } + return false, nil + }); done { + return res } - istore, err := s.ImageStore() - if err != nil { - return false - } - istores, err := s.ROImageStores() - if err != nil { - return false - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return false - } + if done, _ := s.readAllImageStores(func(store roImageStore) (bool, error) { if store.Exists(id) { - return true + res = true + return true, nil } + return false, nil + }); done { + return res } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return false } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return false } - if rcstore.Exists(id) { - return true - } - - return false + defer rcstore.stopReading() + return rcstore.Exists(id) } func dedupeNames(names []string) []string { @@ -2099,149 +2065,95 @@ func (s *store) RemoveNames(id string, names []string) error { func (s *store) updateNames(id string, names []string, op updateNameOperation) error { deduped := dedupeNames(names) - rlstore, err := s.LayerStore() - if err != nil { - return err - } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - if rlstore.Exists(id) { - switch op { - case setNames: - return rlstore.SetNames(id, deduped) - case removeNames: - return rlstore.RemoveNames(id, deduped) - case addNames: - return rlstore.AddNames(id, deduped) - default: - return errInvalidUpdateNameOperation + layerFound := false + if err := s.writeToLayerStore(func(rlstore rwLayerStore) error { + if !rlstore.Exists(id) { + return nil } + layerFound = true + return rlstore.updateNames(id, deduped, op) + }); err != nil || layerFound { + return err } - ristore, err := s.ImageStore() + ristore, err := s.getImageStore() if err != nil { return err } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { + if err := ristore.startWriting(); err != nil { return err } + defer ristore.stopWriting() if ristore.Exists(id) { - switch op { - case setNames: - return ristore.SetNames(id, deduped) - case removeNames: - return ristore.RemoveNames(id, deduped) - case addNames: - return ristore.AddNames(id, deduped) - default: - return errInvalidUpdateNameOperation - } + return ristore.updateNames(id, deduped, op) } // Check is id refers to a RO Store - ristores, err := s.ROImageStores() + ristores, err := s.getROImageStores() if err != nil { return err } for _, s := range ristores { store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { + if err := store.startReading(); err != nil { return err } + defer store.stopReading() if i, err := store.Get(id); err == nil { if len(deduped) > 1 { // Do not want to create image name in R/W storage deduped = deduped[1:] } _, err := ristore.Create(id, deduped, i.TopLayer, i.Metadata, i.Created, i.Digest) - if err == nil { - return ristore.Save() - } return err } } - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - if rcstore.Exists(id) { - switch op { - case setNames: - return rcstore.SetNames(id, deduped) - case removeNames: - return rcstore.RemoveNames(id, deduped) - case addNames: - return rcstore.AddNames(id, deduped) - default: - return errInvalidUpdateNameOperation + containerFound := false + if err := s.writeToContainerStore(func(rcstore rwContainerStore) error { + if !rcstore.Exists(id) { + return nil } + containerFound = true + return rcstore.updateNames(id, deduped, op) + }); err != nil || containerFound { + return err } + return ErrLayerUnknown } func (s *store) Names(id string) ([]string, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res []string + + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if l, err := store.Get(id); l != nil && err == nil { - return l.Names, nil + res = l.Names + return true, nil } + return false, nil + }); done { + return res, err } - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { if i, err := store.Get(id); i != nil && err == nil { - return i.Names, nil + res = i.Names + return true, nil } + return false, nil + }); done { + return res, err } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() if c, err := rcstore.Get(id); c != nil && err == nil { return c.Names, nil } @@ -2249,55 +2161,36 @@ func (s *store) Names(id string) ([]string, error) { } func (s *store) Lookup(name string) (string, error) { - lstore, err := s.LayerStore() - if err != nil { - return "", err - } - lstores, err := s.ROLayerStores() - if err != nil { - return "", err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return "", err - } + var res string + + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if l, err := store.Get(name); l != nil && err == nil { - return l.ID, nil + res = l.ID + return true, nil } + return false, nil + }); done { + return res, err } - istore, err := s.ImageStore() - if err != nil { - return "", err - } - istores, err := s.ROImageStores() - if err != nil { - return "", err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return "", err - } + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { if i, err := store.Get(name); i != nil && err == nil { - return i.ID, nil + res = i.ID + return true, nil } + return false, nil + }); done { + return res, err } - cstore, err := s.ContainerStore() + cstore, err := s.getContainerStore() if err != nil { return "", err } - cstore.RLock() - defer cstore.Unlock() - if err := cstore.ReloadIfChanged(); err != nil { + if err := cstore.startReading(); err != nil { return "", err } + defer cstore.stopReading() if c, err := cstore.Get(name); c != nil && err == nil { return c.ID, nil } @@ -2306,417 +2199,290 @@ func (s *store) Lookup(name string) (string, error) { } func (s *store) DeleteLayer(id string) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - - if rlstore.Exists(id) { - if l, err := rlstore.Get(id); err != nil { - id = l.ID - } - layers, err := rlstore.Layers() - if err != nil { - return err - } - for _, layer := range layers { - if layer.Parent == id { - return fmt.Errorf("used by layer %v: %w", layer.ID, ErrLayerHasChildren) + return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if rlstore.Exists(id) { + if l, err := rlstore.Get(id); err != nil { + id = l.ID } - } - images, err := ristore.Images() - if err != nil { - return err - } - - for _, image := range images { - if image.TopLayer == id { - return fmt.Errorf("layer %v used by image %v: %w", id, image.ID, ErrLayerUsedByImage) + layers, err := rlstore.Layers() + if err != nil { + return err } - if stringutils.InSlice(image.MappedTopLayers, id) { - // No write access to the image store, fail before the layer is deleted - if _, ok := ristore.(*imageStore); !ok { - return fmt.Errorf("layer %v used by image %v: %w", id, image.ID, ErrLayerUsedByImage) + for _, layer := range layers { + if layer.Parent == id { + return fmt.Errorf("used by layer %v: %w", layer.ID, ErrLayerHasChildren) } } - } - containers, err := rcstore.Containers() - if err != nil { - return err - } - for _, container := range containers { - if container.LayerID == id { - return fmt.Errorf("layer %v used by container %v: %w", id, container.ID, ErrLayerUsedByContainer) + images, err := ristore.Images() + if err != nil { + return err } - } - if err := rlstore.Delete(id); err != nil { - return fmt.Errorf("delete layer %v: %w", id, err) - } - // The check here is used to avoid iterating the images if we don't need to. - // There is already a check above for the imageStore to be writeable when the layer is part of MappedTopLayers. - if istore, ok := ristore.(*imageStore); ok { for _, image := range images { + if image.TopLayer == id { + return fmt.Errorf("layer %v used by image %v: %w", id, image.ID, ErrLayerUsedByImage) + } if stringutils.InSlice(image.MappedTopLayers, id) { - if err = istore.removeMappedTopLayer(image.ID, id); err != nil { - return fmt.Errorf("remove mapped top layer %v from image %v: %w", id, image.ID, err) + // No write access to the image store, fail before the layer is deleted + if _, ok := ristore.(*imageStore); !ok { + return fmt.Errorf("layer %v used by image %v: %w", id, image.ID, ErrLayerUsedByImage) } } } + containers, err := rcstore.Containers() + if err != nil { + return err + } + for _, container := range containers { + if container.LayerID == id { + return fmt.Errorf("layer %v used by container %v: %w", id, container.ID, ErrLayerUsedByContainer) + } + } + if err := rlstore.Delete(id); err != nil { + return fmt.Errorf("delete layer %v: %w", id, err) + } + + // The check here is used to avoid iterating the images if we don't need to. + // There is already a check above for the imageStore to be writeable when the layer is part of MappedTopLayers. + if istore, ok := ristore.(*imageStore); ok { + for _, image := range images { + if stringutils.InSlice(image.MappedTopLayers, id) { + if err = istore.removeMappedTopLayer(image.ID, id); err != nil { + return fmt.Errorf("remove mapped top layer %v from image %v: %w", id, image.ID, err) + } + } + } + } + return nil + } + return ErrNotALayer + }) +} + +func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) { + layersToRemove := []string{} + if err := s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if ristore.Exists(id) { + image, err := ristore.Get(id) + if err != nil { + return err + } + id = image.ID + containers, err := rcstore.Containers() + if err != nil { + return err + } + aContainerByImage := make(map[string]string) + for _, container := range containers { + aContainerByImage[container.ImageID] = container.ID + } + if container, ok := aContainerByImage[id]; ok { + return fmt.Errorf("image used by %v: %w", container, ErrImageUsedByContainer) + } + images, err := ristore.Images() + if err != nil { + return err + } + layers, err := rlstore.Layers() + if err != nil { + return err + } + childrenByParent := make(map[string][]string) + for _, layer := range layers { + childrenByParent[layer.Parent] = append(childrenByParent[layer.Parent], layer.ID) + } + otherImagesTopLayers := make(map[string]struct{}) + for _, img := range images { + if img.ID != id { + otherImagesTopLayers[img.TopLayer] = struct{}{} + for _, layerID := range img.MappedTopLayers { + otherImagesTopLayers[layerID] = struct{}{} + } + } + } + if commit { + if err = ristore.Delete(id); err != nil { + return err + } + } + layer := image.TopLayer + layersToRemoveMap := make(map[string]struct{}) + layersToRemove = append(layersToRemove, image.MappedTopLayers...) + for _, mappedTopLayer := range image.MappedTopLayers { + layersToRemoveMap[mappedTopLayer] = struct{}{} + } + for layer != "" { + if rcstore.Exists(layer) { + break + } + if _, used := otherImagesTopLayers[layer]; used { + break + } + parent := "" + if l, err := rlstore.Get(layer); err == nil { + parent = l.Parent + } + hasChildrenNotBeingRemoved := func() bool { + layersToCheck := []string{layer} + if layer == image.TopLayer { + layersToCheck = append(layersToCheck, image.MappedTopLayers...) + } + for _, layer := range layersToCheck { + if childList := childrenByParent[layer]; len(childList) > 0 { + for _, child := range childList { + if _, childIsSlatedForRemoval := layersToRemoveMap[child]; childIsSlatedForRemoval { + continue + } + return true + } + } + } + return false + } + if hasChildrenNotBeingRemoved() { + break + } + layersToRemove = append(layersToRemove, layer) + layersToRemoveMap[layer] = struct{}{} + layer = parent + } + } else { + return ErrNotAnImage + } + if commit { + for _, layer := range layersToRemove { + if err = rlstore.Delete(layer); err != nil { + return err + } + } } return nil - } - return ErrNotALayer -} - -func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) { - rlstore, err := s.LayerStore() - if err != nil { + }); err != nil { return nil, err } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } - rcstore, err := s.ContainerStore() - if err != nil { - return nil, err - } - - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return nil, err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return nil, err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return nil, err - } - layersToRemove := []string{} - if ristore.Exists(id) { - image, err := ristore.Get(id) - if err != nil { - return nil, err - } - id = image.ID - containers, err := rcstore.Containers() - if err != nil { - return nil, err - } - aContainerByImage := make(map[string]string) - for _, container := range containers { - aContainerByImage[container.ImageID] = container.ID - } - if container, ok := aContainerByImage[id]; ok { - return nil, fmt.Errorf("image used by %v: %w", container, ErrImageUsedByContainer) - } - images, err := ristore.Images() - if err != nil { - return nil, err - } - layers, err := rlstore.Layers() - if err != nil { - return nil, err - } - childrenByParent := make(map[string][]string) - for _, layer := range layers { - childrenByParent[layer.Parent] = append(childrenByParent[layer.Parent], layer.ID) - } - otherImagesTopLayers := make(map[string]struct{}) - for _, img := range images { - if img.ID != id { - otherImagesTopLayers[img.TopLayer] = struct{}{} - for _, layerID := range img.MappedTopLayers { - otherImagesTopLayers[layerID] = struct{}{} - } - } - } - if commit { - if err = ristore.Delete(id); err != nil { - return nil, err - } - } - layer := image.TopLayer - layersToRemoveMap := make(map[string]struct{}) - layersToRemove = append(layersToRemove, image.MappedTopLayers...) - for _, mappedTopLayer := range image.MappedTopLayers { - layersToRemoveMap[mappedTopLayer] = struct{}{} - } - for layer != "" { - if rcstore.Exists(layer) { - break - } - if _, used := otherImagesTopLayers[layer]; used { - break - } - parent := "" - if l, err := rlstore.Get(layer); err == nil { - parent = l.Parent - } - hasChildrenNotBeingRemoved := func() bool { - layersToCheck := []string{layer} - if layer == image.TopLayer { - layersToCheck = append(layersToCheck, image.MappedTopLayers...) - } - for _, layer := range layersToCheck { - if childList := childrenByParent[layer]; len(childList) > 0 { - for _, child := range childList { - if _, childIsSlatedForRemoval := layersToRemoveMap[child]; childIsSlatedForRemoval { - continue - } - return true - } - } - } - return false - } - if hasChildrenNotBeingRemoved() { - break - } - layersToRemove = append(layersToRemove, layer) - layersToRemoveMap[layer] = struct{}{} - layer = parent - } - } else { - return nil, ErrNotAnImage - } - if commit { - for _, layer := range layersToRemove { - if err = rlstore.Delete(layer); err != nil { - return nil, err - } - } - } return layersToRemove, nil } func (s *store) DeleteContainer(id string) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rcstore, err := s.ContainerStore() - if err != nil { - return err - } + return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if !rcstore.Exists(id) { + return ErrNotAContainer + } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } + container, err := rcstore.Get(id) + if err != nil { + return ErrNotAContainer + } - if rcstore.Exists(id) { - if container, err := rcstore.Get(id); err == nil { - errChan := make(chan error) - var wg sync.WaitGroup + errChan := make(chan error) + var wg sync.WaitGroup - if rlstore.Exists(container.LayerID) { - wg.Add(1) - go func() { - errChan <- rlstore.Delete(container.LayerID) - wg.Done() - }() - } + if rlstore.Exists(container.LayerID) { wg.Add(1) go func() { - errChan <- rcstore.Delete(id) + errChan <- rlstore.Delete(container.LayerID) wg.Done() }() - - middleDir := s.graphDriverName + "-containers" - gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID) - wg.Add(1) - go func() { - defer wg.Done() - // attempt a simple rm -rf first - err := os.RemoveAll(gcpath) - if err == nil { - errChan <- nil - return - } - // and if it fails get to the more complicated cleanup - errChan <- system.EnsureRemoveAll(gcpath) - }() - - rcpath := filepath.Join(s.RunRoot(), middleDir, container.ID) - wg.Add(1) - go func() { - defer wg.Done() - // attempt a simple rm -rf first - err := os.RemoveAll(rcpath) - if err == nil { - errChan <- nil - return - } - // and if it fails get to the more complicated cleanup - errChan <- system.EnsureRemoveAll(rcpath) - }() - - go func() { - wg.Wait() - close(errChan) - }() - - var errors []error - for err := range errChan { - if err != nil { - errors = append(errors, err) - } - } - return multierror.Append(nil, errors...).ErrorOrNil() } - } - return ErrNotAContainer + wg.Add(1) + go func() { + errChan <- rcstore.Delete(id) + wg.Done() + }() + + middleDir := s.graphDriverName + "-containers" + gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID) + wg.Add(1) + go func() { + defer wg.Done() + // attempt a simple rm -rf first + err := os.RemoveAll(gcpath) + if err == nil { + errChan <- nil + return + } + // and if it fails get to the more complicated cleanup + errChan <- system.EnsureRemoveAll(gcpath) + }() + + rcpath := filepath.Join(s.RunRoot(), middleDir, container.ID) + wg.Add(1) + go func() { + defer wg.Done() + // attempt a simple rm -rf first + err := os.RemoveAll(rcpath) + if err == nil { + errChan <- nil + return + } + // and if it fails get to the more complicated cleanup + errChan <- system.EnsureRemoveAll(rcpath) + }() + + go func() { + wg.Wait() + close(errChan) + }() + + var errors []error + for err := range errChan { + if err != nil { + errors = append(errors, err) + } + } + return multierror.Append(nil, errors...).ErrorOrNil() + }) } func (s *store) Delete(id string) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - - if rcstore.Exists(id) { - if container, err := rcstore.Get(id); err == nil { - if rlstore.Exists(container.LayerID) { - if err = rlstore.Delete(container.LayerID); err != nil { - return err + return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if rcstore.Exists(id) { + if container, err := rcstore.Get(id); err == nil { + if rlstore.Exists(container.LayerID) { + if err = rlstore.Delete(container.LayerID); err != nil { + return err + } + if err = rcstore.Delete(id); err != nil { + return err + } + middleDir := s.graphDriverName + "-containers" + gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID, "userdata") + if err = os.RemoveAll(gcpath); err != nil { + return err + } + rcpath := filepath.Join(s.RunRoot(), middleDir, container.ID, "userdata") + if err = os.RemoveAll(rcpath); err != nil { + return err + } + return nil } - if err = rcstore.Delete(id); err != nil { - return err - } - middleDir := s.graphDriverName + "-containers" - gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID, "userdata") - if err = os.RemoveAll(gcpath); err != nil { - return err - } - rcpath := filepath.Join(s.RunRoot(), middleDir, container.ID, "userdata") - if err = os.RemoveAll(rcpath); err != nil { - return err - } - return nil + return ErrNotALayer } - return ErrNotALayer } - } - if ristore.Exists(id) { - return ristore.Delete(id) - } - if rlstore.Exists(id) { - return rlstore.Delete(id) - } - return ErrLayerUnknown + if ristore.Exists(id) { + return ristore.Delete(id) + } + if rlstore.Exists(id) { + return rlstore.Delete(id) + } + return ErrLayerUnknown + }) } func (s *store) Wipe() error { - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rlstore, err := s.LayerStore() - if err != nil { - return err - } - - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return err - } - ristore.Lock() - defer ristore.Unlock() - if err := ristore.ReloadIfChanged(); err != nil { - return err - } - rcstore.Lock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { - return err - } - - if err = rcstore.Wipe(); err != nil { - return err - } - if err = ristore.Wipe(); err != nil { - return err - } - return rlstore.Wipe() + return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + if err := rcstore.Wipe(); err != nil { + return err + } + if err := ristore.Wipe(); err != nil { + return err + } + return rlstore.Wipe() + }) } func (s *store) Status() ([][2]string, error) { - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return nil, err } @@ -2728,18 +2494,17 @@ func (s *store) Version() ([][2]string, error) { } func (s *store) mount(id string, options drivers.MountOpts) (string, error) { - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return "", err } s.graphLock.Lock() defer s.graphLock.Unlock() - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return "", err } + defer rlstore.stopWriting() modified, err := s.graphLock.Modified() if err != nil { @@ -2811,15 +2576,14 @@ func (s *store) Mounted(id string) (int, error) { if layerID, err := s.ContainerLayerID(id); err == nil { id = layerID } - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return 0, err } - rlstore.RLock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startReading(); err != nil { return 0, err } + defer rlstore.stopReading() return rlstore.Mounted(id) } @@ -2836,73 +2600,50 @@ func (s *store) Unmount(id string, force bool) (bool, error) { if layerID, err := s.ContainerLayerID(id); err == nil { id = layerID } - rlstore, err := s.LayerStore() - if err != nil { - return false, err - } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return false, err - } - if rlstore.Exists(id) { - return rlstore.Unmount(id, force) - } - return false, ErrLayerUnknown + var res bool + err := s.writeToLayerStore(func(rlstore rwLayerStore) error { + if rlstore.Exists(id) { + var err error + res, err = rlstore.Unmount(id, force) + return err + } + return ErrLayerUnknown + }) + return res, err } func (s *store) Changes(from, to string) ([]archive.Change, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res []archive.Change + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if store.Exists(to) { - return store.Changes(from, to) + var err error + res, err = store.Changes(from, to) + return true, err } + return false, nil + }); done { + return res, err } return nil, ErrLayerUnknown } func (s *store) DiffSize(from, to string) (int64, error) { - lstore, err := s.LayerStore() - if err != nil { - return -1, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return -1, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return -1, err - } + var res int64 = -1 + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if store.Exists(to) { - return store.DiffSize(from, to) + var err error + res, err = store.DiffSize(from, to) + return true, err } + return false, nil + }); done { + return res, err } return -1, ErrLayerUnknown } func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() + layerStores, err := s.allLayerStores() if err != nil { return nil, err } @@ -2928,11 +2669,9 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro s.lastLoaded = time.Now() } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range layerStores { store := s - store.RLock() - if err := store.ReloadIfChanged(); err != nil { - store.Unlock() + if err := store.startReading(); err != nil { return nil, err } if store.Exists(to) { @@ -2940,130 +2679,87 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro if rc != nil && err == nil { wrapped := ioutils.NewReadCloserWrapper(rc, func() error { err := rc.Close() - store.Unlock() + store.stopReading() return err }) return wrapped, nil } - store.Unlock() + store.stopReading() return rc, err } - store.Unlock() + store.stopReading() } return nil, ErrLayerUnknown } func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - if err = rlstore.Load(); err != nil { - return err + return s.writeToLayerStore(func(rlstore rwLayerStore) error { + if !rlstore.Exists(to) { + return ErrLayerUnknown } - } - if !rlstore.Exists(to) { - return ErrLayerUnknown - } - return rlstore.ApplyDiffFromStagingDirectory(to, stagingDirectory, diffOutput, options) + return rlstore.ApplyDiffFromStagingDirectory(to, stagingDirectory, diffOutput, options) + }) } func (s *store) CleanupStagingDirectory(stagingDirectory string) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - if err = rlstore.Load(); err != nil { - return err - } - } - return rlstore.CleanupStagingDirectory(stagingDirectory) + return s.writeToLayerStore(func(rlstore rwLayerStore) error { + return rlstore.CleanupStagingDirectory(stagingDirectory) + }) } func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - if err = rlstore.Load(); err != nil { - return nil, err + var res *drivers.DriverWithDifferOutput + err := s.writeToLayerStore(func(rlstore rwLayerStore) error { + if to != "" && !rlstore.Exists(to) { + return ErrLayerUnknown } - } - if to != "" && !rlstore.Exists(to) { - return nil, ErrLayerUnknown - } - return rlstore.ApplyDiffWithDiffer(to, options, differ) + var err error + res, err = rlstore.ApplyDiffWithDiffer(to, options, differ) + return err + }) + return res, err } func (s *store) DifferTarget(id string) (string, error) { - rlstore, err := s.LayerStore() - if err != nil { - return "", err - } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - if err = rlstore.Load(); err != nil { - return "", err + var res string + err := s.writeToLayerStore(func(rlstore rwLayerStore) error { + if rlstore.Exists(id) { + var err error + res, err = rlstore.DifferTarget(id) + return err } - } - if rlstore.Exists(id) { - return rlstore.DifferTarget(id) - } - return "", ErrLayerUnknown + return ErrLayerUnknown + }) + return res, err } func (s *store) ApplyDiff(to string, diff io.Reader) (int64, error) { - rlstore, err := s.LayerStore() - if err != nil { - return -1, err - } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { - return -1, err - } - if rlstore.Exists(to) { - return rlstore.ApplyDiff(to, diff) - } - return -1, ErrLayerUnknown + var res int64 = -1 + err := s.writeToLayerStore(func(rlstore rwLayerStore) error { + if rlstore.Exists(to) { + var err error + res, err = rlstore.ApplyDiff(to, diff) + return err + } + return ErrLayerUnknown + }) + return res, err } -func (s *store) layersByMappedDigest(m func(ROLayerStore, digest.Digest) ([]Layer, error), d digest.Digest) ([]Layer, error) { +func (s *store) layersByMappedDigest(m func(roLayerStore, digest.Digest) ([]Layer, error), d digest.Digest) ([]Layer, error) { var layers []Layer - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + if _, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { storeLayers, err := m(store, d) if err != nil { if !errors.Is(err, ErrLayerUnknown) { - return nil, err + return true, err } - continue + return false, nil } layers = append(layers, storeLayers...) + return false, nil + }); err != nil { + return nil, err } if len(layers) == 0 { return nil, ErrLayerUnknown @@ -3075,49 +2771,40 @@ func (s *store) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) { if err := d.Validate(); err != nil { return nil, fmt.Errorf("looking for compressed layers matching digest %q: %w", d, err) } - return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByCompressedDigest(d) }, d) + return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByCompressedDigest(d) }, d) } func (s *store) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { if err := d.Validate(); err != nil { return nil, fmt.Errorf("looking for layers matching digest %q: %w", d, err) } - return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d) + return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d) } func (s *store) LayerSize(id string) (int64, error) { - lstore, err := s.LayerStore() - if err != nil { - return -1, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return -1, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return -1, err - } + var res int64 = -1 + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { if store.Exists(id) { - return store.Size(id) + var err error + res, err = store.Size(id) + return true, err } + return false, nil + }); done { + return res, err } return -1, ErrLayerUnknown } func (s *store) LayerParentOwners(id string) ([]int, []int, error) { - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return nil, nil, err } - rlstore.RLock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startReading(); err != nil { return nil, nil, err } + defer rlstore.stopReading() if rlstore.Exists(id) { return rlstore.ParentOwners(id) } @@ -3125,24 +2812,22 @@ func (s *store) LayerParentOwners(id string) ([]int, []int, error) { } func (s *store) ContainerParentOwners(id string) ([]int, []int, error) { - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return nil, nil, err } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, nil, err } - rlstore.RLock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startReading(); err != nil { return nil, nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + defer rlstore.stopReading() + if err := rcstore.startReading(); err != nil { return nil, nil, err } + defer rcstore.stopReading() container, err := rcstore.Get(id) if err != nil { return nil, nil, err @@ -3154,106 +2839,60 @@ func (s *store) ContainerParentOwners(id string) ([]int, []int, error) { } func (s *store) Layers() ([]Layer, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - - layers, err := func() ([]Layer, error) { - lstore.Lock() - defer lstore.Unlock() - if err := lstore.Load(); err != nil { - return nil, err - } - return lstore.Layers() - }() - if err != nil { - return nil, err - } - - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - - for _, s := range lstores { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var layers []Layer + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { storeLayers, err := store.Layers() if err != nil { - return nil, err + return true, err } layers = append(layers, storeLayers...) + return false, nil + }); done { + return nil, err } return layers, nil } func (s *store) Images() ([]Image, error) { var images []Image - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + if _, err := s.readAllImageStores(func(store roImageStore) (bool, error) { storeImages, err := store.Images() if err != nil { - return nil, err + return true, err } images = append(images, storeImages...) + return false, nil + }); err != nil { + return nil, err } return images, nil } func (s *store) Containers() ([]Container, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() return rcstore.Containers() } func (s *store) Layer(id string) (*Layer, error) { - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROLayerStore{lstore}, lstores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res *Layer + if done, err := s.readAllLayerStores(func(store roLayerStore) (bool, error) { layer, err := store.Get(id) if err == nil { - return layer, nil + res = layer + return true, nil } + return false, nil + }); done { + return res, err } return nil, ErrLayerUnknown } @@ -3298,29 +2937,27 @@ func (al *additionalLayer) CompressedSize() int64 { } func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) { - rlstore, err := al.s.LayerStore() + rlstore, err := al.s.getLayerStore() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return nil, err } - rlstores, err := al.s.ROLayerStores() + defer rlstore.stopWriting() + rlstores, err := al.s.getROLayerStores() if err != nil { return nil, err } var parentLayer *Layer if parent != "" { - for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) { + for _, lstore := range append([]roLayerStore{rlstore}, rlstores...) { if lstore != rlstore { - lstore.RLock() - defer lstore.Unlock() - if err := lstore.ReloadIfChanged(); err != nil { + if err := lstore.startReading(); err != nil { return nil, err } + defer lstore.stopReading() } parentLayer, err = lstore.Get(parent) if err == nil { @@ -3340,116 +2977,82 @@ func (al *additionalLayer) Release() { } func (s *store) Image(id string) (*Image, error) { - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + var res *Image + if done, err := s.readAllImageStores(func(store roImageStore) (bool, error) { image, err := store.Get(id) if err == nil { - return image, nil + res = image + return true, nil } + return false, nil + }); done { + return res, err } return nil, fmt.Errorf("locating image with ID %q: %w", id, ErrImageUnknown) } func (s *store) ImagesByTopLayer(id string) ([]*Image, error) { - images := []*Image{} layer, err := s.Layer(id) if err != nil { return nil, err } - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, s := range append([]ROImageStore{istore}, istores...) { - store := s - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + images := []*Image{} + if _, err := s.readAllImageStores(func(store roImageStore) (bool, error) { imageList, err := store.Images() if err != nil { - return nil, err + return true, err } for _, image := range imageList { + image := image if image.TopLayer == layer.ID || stringutils.InSlice(image.MappedTopLayers, layer.ID) { images = append(images, &image) } } + return false, nil + }); err != nil { + return nil, err } return images, nil } func (s *store) ImagesByDigest(d digest.Digest) ([]*Image, error) { images := []*Image{} - - istore, err := s.ImageStore() - if err != nil { - return nil, err - } - - istores, err := s.ROImageStores() - if err != nil { - return nil, err - } - for _, store := range append([]ROImageStore{istore}, istores...) { - store.RLock() - defer store.Unlock() - if err := store.ReloadIfChanged(); err != nil { - return nil, err - } + if _, err := s.readAllImageStores(func(store roImageStore) (bool, error) { imageList, err := store.ByDigest(d) if err != nil && !errors.Is(err, ErrImageUnknown) { - return nil, err + return true, err } images = append(images, imageList...) + return false, nil + }); err != nil { + return nil, err } return images, nil } func (s *store) Container(id string) (*Container, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() return rcstore.Get(id) } func (s *store) ContainerLayerID(id string) (string, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return "", err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return "", err } + defer rcstore.stopReading() container, err := rcstore.Get(id) if err != nil { return "", err @@ -3462,15 +3065,14 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { if err != nil { return nil, err } - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return nil, err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return nil, err } + defer rcstore.stopReading() containerList, err := rcstore.Containers() if err != nil { return nil, err @@ -3485,15 +3087,14 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { } func (s *store) ContainerDirectory(id string) (string, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return "", err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return "", err } + defer rcstore.stopReading() id, err = rcstore.Lookup(id) if err != nil { @@ -3509,16 +3110,15 @@ func (s *store) ContainerDirectory(id string) (string, error) { } func (s *store) ContainerRunDirectory(id string) (string, error) { - rcstore, err := s.ContainerStore() + rcstore, err := s.getContainerStore() if err != nil { return "", err } - rcstore.RLock() - defer rcstore.Unlock() - if err := rcstore.ReloadIfChanged(); err != nil { + if err := rcstore.startReading(); err != nil { return "", err } + defer rcstore.stopReading() id, err = rcstore.Lookup(id) if err != nil { @@ -3575,9 +3175,8 @@ func (s *store) FromContainerRunDirectory(id, file string) ([]byte, error) { func (s *store) Shutdown(force bool) ([]string, error) { mounted := []string{} - modified := false - rlstore, err := s.LayerStore() + rlstore, err := s.getLayerStore() if err != nil { return mounted, err } @@ -3585,11 +3184,10 @@ func (s *store) Shutdown(force bool) ([]string, error) { s.graphLock.Lock() defer s.graphLock.Unlock() - rlstore.Lock() - defer rlstore.Unlock() - if err := rlstore.ReloadIfChanged(); err != nil { + if err := rlstore.startWriting(); err != nil { return nil, err } + defer rlstore.stopWriting() layers, err := rlstore.Layers() if err != nil { @@ -3609,7 +3207,6 @@ func (s *store) Shutdown(force bool) ([]string, error) { } break } - modified = true } } } @@ -3618,11 +3215,13 @@ func (s *store) Shutdown(force bool) ([]string, error) { } if err == nil { err = s.graphDriver.Cleanup() - s.graphLock.Touch() - modified = true - } - if modified { - rlstore.Touch() + if err2 := s.graphLock.Touch(); err2 != nil { + if err == nil { + err = err2 + } else { + err = fmt.Errorf("(graphLock.Touch failed: %v) %w", err2, err) + } + } } return mounted, err } @@ -3710,9 +3309,10 @@ const AutoUserNsMaxSize = 65536 // creating a user namespace. const RootAutoUserNsUser = "containers" -// SetDefaultConfigFilePath sets the default configuration to the specified path +// SetDefaultConfigFilePath sets the default configuration to the specified path, and loads the file. +// Deprecated: Use types.SetDefaultConfigFilePath, which can return an error. func SetDefaultConfigFilePath(path string) { - types.SetDefaultConfigFilePath(path) + _ = types.SetDefaultConfigFilePath(path) } // DefaultConfigFile returns the path to the storage config file used @@ -3722,8 +3322,9 @@ func DefaultConfigFile(rootless bool) (string, error) { // ReloadConfigurationFile parses the specified configuration file and overrides // the configuration in storeOptions. +// Deprecated: Use types.ReloadConfigurationFile, which can return an error. func ReloadConfigurationFile(configFile string, storeOptions *types.StoreOptions) { - types.ReloadConfigurationFile(configFile, storeOptions) + _ = types.ReloadConfigurationFile(configFile, storeOptions) } // GetDefaultMountOptions returns the default mountoptions defined in container/storage diff --git a/vendor/github.com/containers/storage/types/options.go b/vendor/github.com/containers/storage/types/options.go index 4c873b45..45daa24e 100644 --- a/vendor/github.com/containers/storage/types/options.go +++ b/vendor/github.com/containers/storage/types/options.go @@ -35,6 +35,14 @@ const ( var ( defaultStoreOptionsOnce sync.Once loadDefaultStoreOptionsErr error + once sync.Once + storeOptions StoreOptions + storeError error + defaultConfigFileSet bool + // defaultConfigFile path to the system wide storage.conf file + defaultConfigFile = SystemConfigFile + // DefaultStoreOptions is a reasonable default set of options. + defaultStoreOptions StoreOptions ) func loadDefaultStoreOptions() { @@ -167,8 +175,8 @@ func defaultStoreOptionsIsolated(rootless bool, rootlessUID int, storageConf str return storageOpts, nil } -// DefaultStoreOptions returns the default storage ops for containers -func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { +// loadStoreOptions returns the default storage ops for containers +func loadStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { storageConf, err := DefaultConfigFile(rootless && rootlessUID != 0) if err != nil { return defaultStoreOptions, err @@ -176,6 +184,21 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { return defaultStoreOptionsIsolated(rootless, rootlessUID, storageConf) } +// UpdateOptions should be called iff container engine recieved a SIGHUP, +// otherwise use DefaultStoreOptions +func UpdateStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { + storeOptions, storeError = loadStoreOptions(rootless, rootlessUID) + return storeOptions, storeError +} + +// DefaultStoreOptions returns the default storage ops for containers +func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { + once.Do(func() { + storeOptions, storeError = loadStoreOptions(rootless, rootlessUID) + }) + return storeOptions, storeError +} + // StoreOptions is used for passing initialization options to GetStore(), for // initializing a Store object and the underlying storage that it controls. type StoreOptions struct { @@ -336,7 +359,7 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) erro } } else { if !os.IsNotExist(err) { - fmt.Printf("Failed to read %s %v\n", configFile, err.Error()) + logrus.Warningf("Failed to read %s %v\n", configFile, err.Error()) return err } } @@ -399,7 +422,7 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) erro if config.Storage.Options.RemapUser != "" && config.Storage.Options.RemapGroup != "" { mappings, err := idtools.NewIDMappings(config.Storage.Options.RemapUser, config.Storage.Options.RemapGroup) if err != nil { - fmt.Printf("Error initializing ID mappings for %s:%s %v\n", config.Storage.Options.RemapUser, config.Storage.Options.RemapGroup, err) + logrus.Warningf("Error initializing ID mappings for %s:%s %v\n", config.Storage.Options.RemapUser, config.Storage.Options.RemapGroup, err) return err } storeOptions.UIDMap = mappings.UIDs() diff --git a/vendor/github.com/containers/storage/types/options_darwin.go b/vendor/github.com/containers/storage/types/options_darwin.go index d5ad50bc..eed1a3d9 100644 --- a/vendor/github.com/containers/storage/types/options_darwin.go +++ b/vendor/github.com/containers/storage/types/options_darwin.go @@ -5,13 +5,9 @@ const ( // for rootless path is constructed via getRootlessStorageOpts defaultRunRoot string = "/run/containers/storage" defaultGraphRoot string = "/var/lib/containers/storage" + SystemConfigFile = "/usr/share/containers/storage.conf" ) -// defaultConfigFile path to the system wide storage.conf file var ( - defaultConfigFile = "/usr/share/containers/storage.conf" defaultOverrideConfigFile = "/etc/containers/storage.conf" - defaultConfigFileSet = false - // DefaultStoreOptions is a reasonable default set of options. - defaultStoreOptions StoreOptions ) diff --git a/vendor/github.com/containers/storage/types/options_freebsd.go b/vendor/github.com/containers/storage/types/options_freebsd.go index d5976b6d..afb7ec6b 100644 --- a/vendor/github.com/containers/storage/types/options_freebsd.go +++ b/vendor/github.com/containers/storage/types/options_freebsd.go @@ -5,13 +5,10 @@ const ( // for rootless path is constructed via getRootlessStorageOpts defaultRunRoot string = "/var/run/containers/storage" defaultGraphRoot string = "/var/db/containers/storage" + SystemConfigFile = "/usr/local/share/containers/storage.conf" ) // defaultConfigFile path to the system wide storage.conf file var ( - defaultConfigFile = "/usr/local/share/containers/storage.conf" defaultOverrideConfigFile = "/usr/local/etc/containers/storage.conf" - defaultConfigFileSet = false - // DefaultStoreOptions is a reasonable default set of options. - defaultStoreOptions StoreOptions ) diff --git a/vendor/github.com/containers/storage/types/options_linux.go b/vendor/github.com/containers/storage/types/options_linux.go index d5ad50bc..d44aaf76 100644 --- a/vendor/github.com/containers/storage/types/options_linux.go +++ b/vendor/github.com/containers/storage/types/options_linux.go @@ -5,13 +5,10 @@ const ( // for rootless path is constructed via getRootlessStorageOpts defaultRunRoot string = "/run/containers/storage" defaultGraphRoot string = "/var/lib/containers/storage" + SystemConfigFile = "/usr/share/containers/storage.conf" ) // defaultConfigFile path to the system wide storage.conf file var ( - defaultConfigFile = "/usr/share/containers/storage.conf" defaultOverrideConfigFile = "/etc/containers/storage.conf" - defaultConfigFileSet = false - // DefaultStoreOptions is a reasonable default set of options. - defaultStoreOptions StoreOptions ) diff --git a/vendor/github.com/containers/storage/types/options_windows.go b/vendor/github.com/containers/storage/types/options_windows.go index d5ad50bc..d44aaf76 100644 --- a/vendor/github.com/containers/storage/types/options_windows.go +++ b/vendor/github.com/containers/storage/types/options_windows.go @@ -5,13 +5,10 @@ const ( // for rootless path is constructed via getRootlessStorageOpts defaultRunRoot string = "/run/containers/storage" defaultGraphRoot string = "/var/lib/containers/storage" + SystemConfigFile = "/usr/share/containers/storage.conf" ) // defaultConfigFile path to the system wide storage.conf file var ( - defaultConfigFile = "/usr/share/containers/storage.conf" defaultOverrideConfigFile = "/etc/containers/storage.conf" - defaultConfigFileSet = false - // DefaultStoreOptions is a reasonable default set of options. - defaultStoreOptions StoreOptions ) diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index c54de763..76ff122b 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -193,7 +193,7 @@ func reloadConfigurationFileIfNeeded(configFile string, storeOptions *StoreOptio fi, err := os.Stat(configFile) if err != nil { if !os.IsNotExist(err) { - fmt.Printf("Failed to read %s %v\n", configFile, err.Error()) + logrus.Warningf("Failed to read %s %v\n", configFile, err.Error()) } return } diff --git a/vendor/github.com/containers/storage/userns.go b/vendor/github.com/containers/storage/userns.go index 6e200ec1..1a3235bc 100644 --- a/vendor/github.com/containers/storage/userns.go +++ b/vendor/github.com/containers/storage/userns.go @@ -123,16 +123,9 @@ func parseMountedFiles(containerMount, passwdFile, groupFile string) uint32 { } // getMaxSizeFromImage returns the maximum ID used by the specified image. -// The layer stores must be already locked. -func (s *store) getMaxSizeFromImage(image *Image, passwdFile, groupFile string) (uint32, error) { - lstore, err := s.LayerStore() - if err != nil { - return 0, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return 0, err - } +// On entry, rlstore must be locked for writing, and lstores must be locked for reading. +func (s *store) getMaxSizeFromImage(image *Image, rlstore rwLayerStore, lstores []roLayerStore, passwdFile, groupFile string) (_ uint32, retErr error) { + layerStores := append([]roLayerStore{rlstore}, lstores...) size := uint32(0) @@ -140,7 +133,7 @@ func (s *store) getMaxSizeFromImage(image *Image, passwdFile, groupFile string) layerName := image.TopLayer outer: for { - for _, ls := range append([]ROLayerStore{lstore}, lstores...) { + for _, ls := range layerStores { layer, err := ls.Get(layerName) if err != nil { continue @@ -167,11 +160,6 @@ outer: return 0, fmt.Errorf("cannot find layer %q", layerName) } - rlstore, err := s.LayerStore() - if err != nil { - return 0, err - } - layerOptions := &LayerOptions{ IDMappingOptions: types.IDMappingOptions{ HostUIDMapping: true, @@ -187,7 +175,15 @@ outer: if err != nil { return 0, err } - defer rlstore.Delete(clayer.ID) + defer func() { + if err2 := rlstore.Delete(clayer.ID); err2 != nil { + if retErr == nil { + retErr = fmt.Errorf("deleting temporary layer %#v: %w", clayer.ID, err2) + } else { + logrus.Errorf("Error deleting temporary layer %#v: %v", clayer.ID, err2) + } + } + }() mountOptions := drivers.MountOpts{ MountLabel: "", @@ -200,7 +196,15 @@ outer: if err != nil { return 0, err } - defer rlstore.Unmount(clayer.ID, true) + defer func() { + if _, err2 := rlstore.Unmount(clayer.ID, true); err2 != nil { + if retErr == nil { + retErr = fmt.Errorf("unmounting temporary layer %#v: %w", clayer.ID, err2) + } else { + logrus.Errorf("Error unmounting temporary layer %#v: %v", clayer.ID, err2) + } + } + }() userFilesSize := parseMountedFiles(mountpoint, passwdFile, groupFile) if userFilesSize > size { @@ -211,7 +215,8 @@ outer: } // getAutoUserNS creates an automatic user namespace -func (s *store) getAutoUserNS(options *types.AutoUserNsOptions, image *Image) ([]idtools.IDMap, []idtools.IDMap, error) { +// If image != nil, On entry, rlstore must be locked for writing, and lstores must be locked for reading. +func (s *store) getAutoUserNS(options *types.AutoUserNsOptions, image *Image, rlstore rwLayerStore, lstores []roLayerStore) ([]idtools.IDMap, []idtools.IDMap, error) { requestedSize := uint32(0) initialSize := uint32(1) if options.Size > 0 { @@ -250,7 +255,7 @@ func (s *store) getAutoUserNS(options *types.AutoUserNsOptions, image *Image) ([ size = s.autoNsMinSize } if image != nil { - sizeFromImage, err := s.getMaxSizeFromImage(image, options.PasswdFile, options.GroupFile) + sizeFromImage, err := s.getMaxSizeFromImage(image, rlstore, lstores, options.PasswdFile, options.GroupFile) if err != nil { return nil, nil, err } diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 3c00c1af..9ec000ff 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -17,6 +17,13 @@ This package provides various compression algorithms. # changelog +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + * Sept 16, 2022 (v1.15.10) * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index f52d1aed..da814715 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -232,7 +232,7 @@ func (b *blockDec) decodeBuf(hist *history) error { if b.lowMem { b.dst = make([]byte, b.RLESize) } else { - b.dst = make([]byte, maxBlockSize) + b.dst = make([]byte, maxCompressedBlockSize) } } b.dst = b.dst[:b.RLESize] diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 78c10755..74d645f7 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -770,7 +770,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch if block.lowMem { block.dst = make([]byte, block.RLESize) } else { - block.dst = make([]byte, maxBlockSize) + block.dst = make([]byte, maxCompressedBlockSize) } } block.dst = block.dst[:block.RLESize] diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 181edc02..202636db 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -304,7 +304,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { minNonLiteralBlockSize = 1 + 1 + inputMargin ) if debugEncoder { - if len(src) > maxBlockSize { + if len(src) > maxCompressedBlockSize { panic("src too big") } } diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index b6c50541..2c0affcd 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -261,11 +261,16 @@ func (d *frameDec) reset(br byteBuffer) error { } d.history.windowSize = int(d.WindowSize) if !d.o.lowMem || d.history.windowSize < maxBlockSize { - // Alloc 2x window size if not low-mem, or very small window size. + // Alloc 2x window size if not low-mem, or window size below 2MB. d.history.allocFrameBuffer = d.history.windowSize * 2 } else { - // Alloc with one additional block - d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize + if d.o.lowMem { + // Alloc with 1MB extra. + d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize/2 + } else { + // Alloc with 2MB extra. + d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize + } } if debugDecoder { diff --git a/vendor/modules.txt b/vendor/modules.txt index 7b079e7f..0ebe27a5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# github.com/BurntSushi/toml v1.2.0 +# github.com/BurntSushi/toml v1.2.1 ## explicit; go 1.16 github.com/BurntSushi/toml github.com/BurntSushi/toml/internal @@ -9,7 +9,7 @@ github.com/Microsoft/go-winio/backuptar github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/pkg/security github.com/Microsoft/go-winio/vhd -# github.com/Microsoft/hcsshim v0.9.4 +# github.com/Microsoft/hcsshim v0.9.5 ## explicit; go 1.13 github.com/Microsoft/hcsshim github.com/Microsoft/hcsshim/computestorage @@ -42,7 +42,7 @@ github.com/acarl005/stripansi # github.com/containerd/cgroups v1.0.4 ## explicit; go 1.17 github.com/containerd/cgroups/stats/v1 -# github.com/containerd/stargz-snapshotter/estargz v0.12.0 +# github.com/containerd/stargz-snapshotter/estargz v0.12.1 ## explicit; go 1.16 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil @@ -136,8 +136,8 @@ github.com/containers/ocicrypt/keywrap/pkcs7 github.com/containers/ocicrypt/spec github.com/containers/ocicrypt/utils github.com/containers/ocicrypt/utils/keyprovider -# github.com/containers/storage v1.43.0 -## explicit; go 1.16 +# github.com/containers/storage v1.44.0 +## explicit; go 1.17 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -275,7 +275,7 @@ github.com/inconshreveable/mousetrap # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.15.11 +# github.com/klauspost/compress v1.15.12 ## explicit; go 1.17 github.com/klauspost/compress github.com/klauspost/compress/flate