From f5a7175f923281ba8a0d3681f117c1eb27728a1f Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 28 Sep 2020 15:44:53 -0500 Subject: [PATCH 001/124] runtime: cloud-hypervisor: tag openapi-generator-cli container Tag openapi-generator-cli container to v4.3.1 that is the latest stable, this way we can have reproducible builds and the same generated code in all the systems Signed-off-by: Julio Montes --- .../pkg/cloud-hypervisor/Makefile | 2 +- .../client/.openapi-generator/FILES | 69 ------------------- .../client/.openapi-generator/VERSION | 2 +- .../cloud-hypervisor/client/api_default.go | 23 ------- .../pkg/cloud-hypervisor/client/client.go | 2 +- 5 files changed, 3 insertions(+), 95 deletions(-) delete mode 100644 src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile b/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile index 2ab3ea54db..73ca84e937 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile @@ -13,7 +13,7 @@ YQ := $(shell command -v yq 2> /dev/null) generate-client-code: clean-generated-code docker run --rm \ --user $$(id -u):$$(id -g) \ - -v $${PWD}:/local openapitools/openapi-generator-cli generate \ + -v $${PWD}:/local openapitools/openapi-generator-cli:v4.3.1 generate \ -i /local/cloud-hypervisor.yaml \ -g go \ -o /local/client diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES deleted file mode 100644 index 2a3283dd04..0000000000 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES +++ /dev/null @@ -1,69 +0,0 @@ -.gitignore -.openapi-generator-ignore -.travis.yml -README.md -api/openapi.yaml -api_default.go -client.go -configuration.go -docs/CmdLineConfig.md -docs/ConsoleConfig.md -docs/CpuTopology.md -docs/CpusConfig.md -docs/DefaultApi.md -docs/DeviceConfig.md -docs/DiskConfig.md -docs/FsConfig.md -docs/InitramfsConfig.md -docs/KernelConfig.md -docs/MemoryConfig.md -docs/MemoryZoneConfig.md -docs/NetConfig.md -docs/NumaConfig.md -docs/NumaDistance.md -docs/PciDeviceInfo.md -docs/PmemConfig.md -docs/RestoreConfig.md -docs/RngConfig.md -docs/SgxEpcConfig.md -docs/VmAddDevice.md -docs/VmConfig.md -docs/VmInfo.md -docs/VmRemoveDevice.md -docs/VmResize.md -docs/VmResizeZone.md -docs/VmSnapshotConfig.md -docs/VmmPingResponse.md -docs/VsockConfig.md -git_push.sh -go.mod -go.sum -model_cmd_line_config.go -model_console_config.go -model_cpu_topology.go -model_cpus_config.go -model_device_config.go -model_disk_config.go -model_fs_config.go -model_initramfs_config.go -model_kernel_config.go -model_memory_config.go -model_memory_zone_config.go -model_net_config.go -model_numa_config.go -model_numa_distance.go -model_pci_device_info.go -model_pmem_config.go -model_restore_config.go -model_rng_config.go -model_sgx_epc_config.go -model_vm_add_device.go -model_vm_config.go -model_vm_info.go -model_vm_remove_device.go -model_vm_resize.go -model_vm_resize_zone.go -model_vm_snapshot_config.go -model_vmm_ping_response.go -model_vsock_config.go -response.go diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/VERSION b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/VERSION index d99e7162d0..ecedc98d1d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/VERSION +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/VERSION @@ -1 +1 @@ -5.0.0-SNAPSHOT \ No newline at end of file +4.3.1 \ No newline at end of file diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go index 83daeae939..16b57bad3c 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go @@ -14,7 +14,6 @@ import ( _ioutil "io/ioutil" _nethttp "net/http" _neturl "net/url" - _bytes "bytes" ) // Linger please @@ -73,7 +72,6 @@ func (a *DefaultApiService) BootVM(ctx _context.Context) (*_nethttp.Response, er localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -140,7 +138,6 @@ func (a *DefaultApiService) CreateVM(ctx _context.Context, vmConfig VmConfig) (* localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -204,7 +201,6 @@ func (a *DefaultApiService) DeleteVM(ctx _context.Context) (*_nethttp.Response, localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -268,7 +264,6 @@ func (a *DefaultApiService) PauseVM(ctx _context.Context) (*_nethttp.Response, e localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -332,7 +327,6 @@ func (a *DefaultApiService) RebootVM(ctx _context.Context) (*_nethttp.Response, localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -396,7 +390,6 @@ func (a *DefaultApiService) ResumeVM(ctx _context.Context) (*_nethttp.Response, localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -460,7 +453,6 @@ func (a *DefaultApiService) ShutdownVM(ctx _context.Context) (*_nethttp.Response localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -524,7 +516,6 @@ func (a *DefaultApiService) ShutdownVMM(ctx _context.Context) (*_nethttp.Respons localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -593,7 +584,6 @@ func (a *DefaultApiService) VmAddDevicePut(ctx _context.Context, vmAddDevice VmA localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -671,7 +661,6 @@ func (a *DefaultApiService) VmAddDiskPut(ctx _context.Context, diskConfig DiskCo localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -749,7 +738,6 @@ func (a *DefaultApiService) VmAddFsPut(ctx _context.Context, fsConfig FsConfig) localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -827,7 +815,6 @@ func (a *DefaultApiService) VmAddNetPut(ctx _context.Context, netConfig NetConfi localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -905,7 +892,6 @@ func (a *DefaultApiService) VmAddPmemPut(ctx _context.Context, pmemConfig PmemCo localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -983,7 +969,6 @@ func (a *DefaultApiService) VmAddVsockPut(ctx _context.Context, vsockConfig Vsoc localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -1058,7 +1043,6 @@ func (a *DefaultApiService) VmCountersGet(ctx _context.Context) (map[string]map[ localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -1133,7 +1117,6 @@ func (a *DefaultApiService) VmInfoGet(ctx _context.Context) (VmInfo, *_nethttp.R localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } @@ -1209,7 +1192,6 @@ func (a *DefaultApiService) VmRemoveDevicePut(ctx _context.Context, vmRemoveDevi localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -1276,7 +1258,6 @@ func (a *DefaultApiService) VmResizePut(ctx _context.Context, vmResize VmResize) localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -1343,7 +1324,6 @@ func (a *DefaultApiService) VmResizeZonePut(ctx _context.Context, vmResizeZone V localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -1410,7 +1390,6 @@ func (a *DefaultApiService) VmRestorePut(ctx _context.Context, restoreConfig Res localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -1477,7 +1456,6 @@ func (a *DefaultApiService) VmSnapshotPut(ctx _context.Context, vmSnapshotConfig localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarHTTPResponse, err } @@ -1543,7 +1521,6 @@ func (a *DefaultApiService) VmmPingGet(ctx _context.Context) (VmmPingResponse, * localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) localVarHTTPResponse.Body.Close() - localVarHTTPResponse.Body = _ioutil.NopCloser(_bytes.NewBuffer(localVarBody)) if err != nil { return localVarReturnValue, localVarHTTPResponse, err } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/client.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/client.go index 40d3ec919e..b435e963f2 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/client.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/client.go @@ -36,7 +36,7 @@ import ( ) var ( - jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?(?:problem\+)?json)`) + jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) ) From 9517b0a9338492f312a3e8240a6cf35c0ac907f7 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 28 Sep 2020 15:48:52 -0500 Subject: [PATCH 002/124] versions: cloud-hypervisor: bump version Use commit c54452c08a467a3e35d8d72f2a91d424e9718c57 as version for cloud-hypervisor. Bring openapi fix cloud-hypervisor/cloud-hypervisor#1760 to support SGX. Signed-off-by: Julio Montes --- .../pkg/cloud-hypervisor/client/api/openapi.yaml | 4 ++-- .../pkg/cloud-hypervisor/client/api_default.go | 6 +++--- .../pkg/cloud-hypervisor/client/docs/DefaultApi.md | 4 ++-- .../pkg/cloud-hypervisor/client/docs/SgxEpcConfig.md | 2 +- .../pkg/cloud-hypervisor/client/model_sgx_epc_config.go | 2 +- .../pkg/cloud-hypervisor/cloud-hypervisor.yaml | 4 ++-- versions.yaml | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index e765d67b31..c9a06aeb9c 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -518,7 +518,7 @@ components: VmCounters: additionalProperties: additionalProperties: - format: uint64 + format: int64 type: integer type: object type: object @@ -1158,7 +1158,7 @@ components: size: 8 properties: size: - format: uint64 + format: int64 type: integer prefault: default: false diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go index 16b57bad3c..216e03e55b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go @@ -996,16 +996,16 @@ func (a *DefaultApiService) VmAddVsockPut(ctx _context.Context, vsockConfig Vsoc /* VmCountersGet Get counters from the VM * @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). -@return map[string]map[string]int32 +@return map[string]map[string]int64 */ -func (a *DefaultApiService) VmCountersGet(ctx _context.Context) (map[string]map[string]int32, *_nethttp.Response, error) { +func (a *DefaultApiService) VmCountersGet(ctx _context.Context) (map[string]map[string]int64, *_nethttp.Response, error) { var ( localVarHTTPMethod = _nethttp.MethodGet localVarPostBody interface{} localVarFormFileName string localVarFileName string localVarFileBytes []byte - localVarReturnValue map[string]map[string]int32 + localVarReturnValue map[string]map[string]int64 ) // create path and map variables diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md index 7d4ed80355..e84fcf38af 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md @@ -451,7 +451,7 @@ No authorization required ## VmCountersGet -> map[string]map[string]int32 VmCountersGet(ctx, ) +> map[string]map[string]int64 VmCountersGet(ctx, ) Get counters from the VM @@ -461,7 +461,7 @@ This endpoint does not need any parameter. ### Return type -[**map[string]map[string]int32**](map.md) +[**map[string]map[string]int64**](map.md) ### Authorization diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SgxEpcConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SgxEpcConfig.md index 929c278c8f..ab045e03e3 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SgxEpcConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/SgxEpcConfig.md @@ -4,7 +4,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- -**Size** | **int32** | | +**Size** | **int64** | | **Prefault** | **bool** | | [optional] [default to false] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_sgx_epc_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_sgx_epc_config.go index 9af3428094..692122c387 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_sgx_epc_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_sgx_epc_config.go @@ -10,6 +10,6 @@ package openapi // SgxEpcConfig struct for SgxEpcConfig type SgxEpcConfig struct { - Size int32 `json:"size"` + Size int64 `json:"size"` Prefault bool `json:"prefault,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 138378127a..88c4b64ffd 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -368,7 +368,7 @@ components: type: object additionalProperties: type: integer - format: uint64 + format: int64 PciDeviceInfo: required: @@ -741,7 +741,7 @@ components: properties: size: type: integer - format: uint64 + format: int64 prefault: type: boolean default: false diff --git a/versions.yaml b/versions.yaml index d90a88eea1..94e592a88c 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v0.10.0" + version: "c54452c08a467a3e35d8d72f2a91d424e9718c57" firecracker: description: "Firecracker micro-VMM" From 1bddde729b35f9bd3de1e1112ffc70aab54ca0c0 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Thu, 1 Oct 2020 12:25:48 -0500 Subject: [PATCH 003/124] ci: add github action to test the snap Add github action to test that the snap package was generated correctly, this CI don't test the snap, it just build it. fixes #838 Signed-off-by: Julio Montes --- .github/workflows/snap.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/snap.yaml diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml new file mode 100644 index 0000000000..40abbde98a --- /dev/null +++ b/.github/workflows/snap.yaml @@ -0,0 +1,15 @@ +name: snap CI +on: ["pull_request"] +jobs: + test: + runs-on: ubuntu-20.04 + steps: + - name: Check out + uses: actions/checkout@v2 + + - name: Install Snapcraft + uses: samuelmeuli/action-snapcraft@v1 + + - name: Build snap + run: | + snapcraft -d snap --destructive-mode From e5acb1257f366991ee43714ac69b82e8cc4ddf4b Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 2 Oct 2020 15:34:39 -0700 Subject: [PATCH 004/124] docs: update dev guide for agent build Include details on setting up rust. Fixes: #851 Signed-off-by: Eric Ernst --- docs/Developer-Guide.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 20293490b1..38fa746e2b 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -75,6 +75,11 @@ You need to install the following to build Kata Containers components: To view the versions of go known to work, see the `golang` entry in the [versions database](../versions.yaml). +- [rust](https://www.rust-lang.org/tools/install) + + To view the versions of rust known to work, see the `rust` entry in the + [versions database](../versions.yaml). + - `make`. - `gcc` (required for building the shim and runtime). @@ -247,6 +252,15 @@ $ sudo systemctl restart systemd-journald > > - You should only do this step if you are testing with the latest version of the agent. +The rust-agent is built with a static linked `musl.` To configure this: + +``` +rustup target add x86_64-unknown-linux-musl +sudo ln -s /usr/bin/g++ /bin/musl-g++ +``` + +To build the agent: + ``` $ go get -d -u github.com/kata-containers/kata-containers $ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make From c488cc48a26fc36720e5d0aee479b8ddbe231d8e Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Thu, 20 Aug 2020 17:11:49 -0700 Subject: [PATCH 005/124] docs: Update docs for enabling agent debug console The systemd method of adding a debug console is not really user friendly. Since we have added a much more straightforward method to enable agent debug console, update developer guide to reflect this. Fixes #834 Signed-off-by: Archana Shinde --- docs/Developer-Guide.md | 85 +++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 38fa746e2b..3b56fa0322 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -41,11 +41,14 @@ * [Connect to debug console](#connect-to-debug-console) * [Traditional debug console setup](#traditional-debug-console-setup) * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) - * [Create a debug systemd service](#create-a-debug-systemd-service) * [Build the debug image](#build-the-debug-image) * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image) + * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) + * [Enabling debug console for QEMU](#enabling-debug-console-for-qemu) + * [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker) * [Create a container](#create-a-container) * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) + * [Obtain details of the image](#obtain-details-of-the-image) * [Capturing kernel boot logs](#capturing-kernel-boot-logs) # Warning @@ -540,35 +543,6 @@ $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/too $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos' ``` -#### Create a debug systemd service - -Create the service file that starts the shell in the rootfs directory: - -``` -$ cat < **Note** Ports 1024 and 1025 are reserved for communication with the agent +> and gathering of agent logs respectively. + +Next, connect to the debug console. The VSOCKS paths vary slightly between +cloud-hypervisor and firecracker. +In case of cloud-hypervisor, connect to the `vsock` as shown: +``` +$ sudo su -c 'cd /var/run/vc/vm/{sandbox_id}/root/ && socat stdin unix-connect:clh.sock' +CONNECT 1026 +``` + +**Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command. + +For firecracker, connect to the `hvsock` as shown: +``` +$ sudo su -c 'cd /var/run/vc/firecracker/{sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock' +CONNECT 1026 ``` **Note**: You need to press the `RETURN` key to see the shell prompt. From de8dcb154927f75ffb4d80a2b8fd25f8dcf105a1 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 2 Oct 2020 16:16:55 -0700 Subject: [PATCH 006/124] dev-guide: update kata-agent install details Install paths were wrong. Updated based on new agent... Signed-off-by: Eric Ernst --- docs/Developer-Guide.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 3b56fa0322..32dd3260db 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -305,9 +305,9 @@ You MUST choose one of `alpine`, `centos`, `clearlinux`, `debian`, `euleros`, `f > - You should only do this step if you are testing with the latest version of the agent. ``` -$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/bin ../../agent/kata-agent -$ sudo install -o root -g root -m 0440 ../../agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ -$ sudo install -o root -g root -m 0440 ../../agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ +$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent +$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ +$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ ``` ### Build a rootfs image From 4ff3ed510117c4684ce19eeb4c140924c2052418 Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Sun, 4 Oct 2020 15:40:48 -0700 Subject: [PATCH 007/124] docs: update networking description First, most people don't care about CNM. Move that out of main doc. Second, tc-filter is the default. Let's add a bit more background on our usage of tc-filter (and clarify why we use this instead of macvtap). Fixes #797 Signed-off-by: Eric Ernst Signed-off-by: Chelsea Mafrica --- docs/design/architecture.md | 66 +++++++++---------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 6291f76027..cb9e0e61df 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -13,7 +13,6 @@ - [Runtime](#runtime) - [Configuration](#configuration) - [Networking](#networking) - - [CNM](#cnm) - [Network Hotplug](#network-hotplug) - [Storage](#storage) - [Kubernetes support](#kubernetes-support) @@ -157,66 +156,31 @@ In order to do so, container engines will usually add one end of a virtual ethernet (`veth`) pair into the container networking namespace. The other end of the `veth` pair is added to the host networking namespace. -This is a very namespace-centric approach as many hypervisors (in particular QEMU) -cannot handle `veth` interfaces. Typically, `TAP` interfaces are created for VM -connectivity. +This is a very namespace-centric approach as many hypervisors/VMMs cannot handle `veth` +interfaces. Typically, `TAP` interfaces are created for VM connectivity. To overcome incompatibility between typical container engines expectations and virtual machines, Kata Containers networking transparently connects `veth` -interfaces with `TAP` ones using MACVTAP: +interfaces with `TAP` ones using Traffic Control: ![Kata Containers networking](arch-images/network.png) +With a TC filter in place, a redirection is created between the container network and the +virtual machine. As an example, the CNI may create a device, `eth0`, in the container's network +namespace, which is a VETH device. Kata Containers will create a tap device for the VM, `kata_tap0`, +and setup a TC redirection filter to mirror traffic from `eth0`'s ingress to `kata_tap0`'s egress, +and a second to mirror traffic from `kata_tap0`'s ingress to `eth0`'s egress. + +Kata Containers maintains support for MACVTAP, which was an earlier implementation used in Kata. TC-filter +is the default because it allows for simpler configuration, better CNI plugin compatibility, and performance +on par with MACVTAP. + +Kata Containers has deprecated support for bridge due to lacking performance relative to TC-filter and MACVTAP. + Kata Containers supports both [CNM](https://github.com/docker/libnetwork/blob/master/docs/design.md#the-container-network-model) and [CNI](https://github.com/containernetworking/cni) for networking management. -### CNM - -![High-level CNM Diagram](arch-images/CNM_overall_diagram.png) - -__CNM lifecycle__ - -1. `RequestPool` - -2. `CreateNetwork` - -3. `RequestAddress` - -4. `CreateEndPoint` - -5. `CreateContainer` - -6. Create `config.json` - -7. Create PID and network namespace - -8. `ProcessExternalKey` - -9. `JoinEndPoint` - -10. `LaunchContainer` - -11. Launch - -12. Run container - -![Detailed CNM Diagram](arch-images/CNM_detailed_diagram.png) - -__Runtime network setup with CNM__ - -1. Read `config.json` - -2. Create the network namespace - -3. Call the `prestart` hook (from inside the netns) - -4. Scan network interfaces inside netns and get the name of the interface - created by prestart hook - -5. Create bridge, TAP, and link all together with network interface previously - created - ### Network Hotplug Kata Containers has developed a set of network sub-commands and APIs to add, list and From 059b89cd039b8bf200c7ae83fb4d0cc33012d65d Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Sun, 4 Oct 2020 15:46:54 -0700 Subject: [PATCH 008/124] docs: Change kata_tap0 to tap0_kata Tap device's should be tap0_kata for architecture.md Fixes #797 Signed-off-by: duanquanfeng Signed-off-by: Chelsea Mafrica --- docs/design/architecture.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index cb9e0e61df..f57057da8b 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -167,9 +167,9 @@ interfaces with `TAP` ones using Traffic Control: With a TC filter in place, a redirection is created between the container network and the virtual machine. As an example, the CNI may create a device, `eth0`, in the container's network -namespace, which is a VETH device. Kata Containers will create a tap device for the VM, `kata_tap0`, -and setup a TC redirection filter to mirror traffic from `eth0`'s ingress to `kata_tap0`'s egress, -and a second to mirror traffic from `kata_tap0`'s ingress to `eth0`'s egress. +namespace, which is a VETH device. Kata Containers will create a tap device for the VM, `tap0_kata`, +and setup a TC redirection filter to mirror traffic from `eth0`'s ingress to `tap0_kata`'s egress, +and a second to mirror traffic from `tap0_kata`'s ingress to `eth0`'s egress. Kata Containers maintains support for MACVTAP, which was an earlier implementation used in Kata. TC-filter is the default because it allows for simpler configuration, better CNI plugin compatibility, and performance From 63c475786fd7cb9e44b306131268c93d267eadc3 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 5 Oct 2020 12:13:36 -0700 Subject: [PATCH 009/124] versions: cloud-hypervisor: Bump to version 6d30fe05 The cloud-hypervisor commit `6d30fe05` introduced a fix on its API for VFIO device hotplug (`VmAddDevice`), which is required for supporting VFIO unplug through openAPI calls in kata. Signed-off-by: Bo Chen --- .../cloud-hypervisor/client/api/openapi.yaml | 19 +++++++++++++------ .../client/docs/MemoryConfig.md | 2 +- .../client/docs/VmAddDevice.md | 2 ++ .../client/model_memory_config.go | 2 +- .../client/model_vm_add_device.go | 2 ++ .../cloud-hypervisor/cloud-hypervisor.yaml | 17 +++++++++++------ versions.yaml | 2 +- 7 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index c9a06aeb9c..e775606d88 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -828,7 +828,7 @@ components: default: false type: boolean host_numa_node: - format: uint32 + format: int32 type: integer hotplug_size: format: int64 @@ -896,7 +896,7 @@ components: default: false type: boolean balloon_size: - format: uint64 + format: int64 type: integer zones: items: @@ -1172,10 +1172,10 @@ components: destination: 3 properties: destination: - format: uint32 + format: int32 type: integer distance: - format: uint8 + format: int32 type: integer required: - destination @@ -1197,11 +1197,11 @@ components: guest_numa_id: 9 properties: guest_numa_id: - format: uint32 + format: int32 type: integer cpus: items: - format: uint8 + format: int32 type: integer type: array distances: @@ -1248,9 +1248,16 @@ components: VmAddDevice: example: path: path + iommu: false + id: id properties: path: type: string + iommu: + default: false + type: boolean + id: + type: string type: object VmRemoveDevice: example: diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md index ccafc86a73..c8dee09c48 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/MemoryConfig.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **Shared** | **bool** | | [optional] [default to false] **Hugepages** | **bool** | | [optional] [default to false] **Balloon** | **bool** | | [optional] [default to false] -**BalloonSize** | **int32** | | [optional] +**BalloonSize** | **int64** | | [optional] **Zones** | [**[]MemoryZoneConfig**](MemoryZoneConfig.md) | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md index e281db1380..124ac3b5d6 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmAddDevice.md @@ -5,6 +5,8 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **Path** | **string** | | [optional] +**Iommu** | **bool** | | [optional] [default to false] +**Id** | **string** | | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go index b0ccbc0348..749080979d 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_memory_config.go @@ -18,6 +18,6 @@ type MemoryConfig struct { Shared bool `json:"shared,omitempty"` Hugepages bool `json:"hugepages,omitempty"` Balloon bool `json:"balloon,omitempty"` - BalloonSize int32 `json:"balloon_size,omitempty"` + BalloonSize int64 `json:"balloon_size,omitempty"` Zones []MemoryZoneConfig `json:"zones,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go index ee96b9b9cd..f644fa60a2 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_add_device.go @@ -11,4 +11,6 @@ package openapi // VmAddDevice struct for VmAddDevice type VmAddDevice struct { Path string `json:"path,omitempty"` + Iommu bool `json:"iommu,omitempty"` + Id string `json:"id,omitempty"` } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index 88c4b64ffd..240e02e27b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -492,7 +492,7 @@ components: default: false host_numa_node: type: integer - format: uint32 + format: int32 hotplug_size: type: integer format: int64 @@ -532,7 +532,7 @@ components: default: false balloon_size: type: integer - format: uint64 + format: int64 zones: type: array items: @@ -754,10 +754,10 @@ components: properties: destination: type: integer - format: uint32 + format: int32 distance: type: integer - format: uint8 + format: int32 NumaConfig: required: @@ -766,12 +766,12 @@ components: properties: guest_numa_id: type: integer - format: uint32 + format: int32 cpus: type: array items: type: integer - format: uint8 + format: int32 distances: type: array items: @@ -811,6 +811,11 @@ components: properties: path: type: string + iommu: + type: boolean + default: false + id: + type: string VmRemoveDevice: type: object diff --git a/versions.yaml b/versions.yaml index 94e592a88c..5be57d05e0 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "c54452c08a467a3e35d8d72f2a91d424e9718c57" + version: "6d30fe05e4febd930d91bb36294f0219faf2254c" firecracker: description: "Firecracker micro-VMM" From 47cfeaaf186eb826160946ae99975bd348a956cb Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 29 Sep 2020 11:03:09 -0700 Subject: [PATCH 010/124] clh: Remove unnecessary VmmPing We can rely on the error handling of the actual HTTP API calls to catch errors, and don't need to call VmmPing explicitly in advance. Signed-off-by: Bo Chen --- src/runtime/virtcontainers/clh.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 9303e44956..15e10ba380 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -420,15 +420,12 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro " using '%v' but only support '%v'", clh.config.BlockDeviceDriver, config.VirtioBlock) } + var err error + cl := clh.client() ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmmPingGet(ctx) - if err != nil { - return openAPIClientError(err) - } - driveID := clhDriveIndexToID(drive.Index) //Explicitly set PCIAddr to NULL, so that VirtPath can be used @@ -457,12 +454,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmmPingGet(ctx) - if err != nil { - return openAPIClientError(err) - } - - _, _, err = cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) + _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) if err != nil { err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } From ace6f1e66e81d9b56a694be93a4cc87f0ab2a27c Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 29 Sep 2020 17:03:44 -0700 Subject: [PATCH 011/124] clh: Support VFIO device unplug This patch adds the support of VFIO device unplug when using cloud-hypervisor. Fixes: #860 Signed-off-by: Bo Chen --- src/runtime/virtcontainers/clh.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 15e10ba380..3ac5ddba76 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -454,7 +454,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device config.VFIODev) error { ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) defer cancel() - _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev}) + _, _, err := cl.VmAddDevicePut(ctx, chclient.VmAddDevice{Path: device.SysfsDev, Id: device.ID}) if err != nil { err = fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } @@ -498,6 +498,20 @@ func (clh *cloudHypervisor) hotplugRemoveBlockDevice(drive *config.BlockDrive) e return err } +func (clh *cloudHypervisor) hotplugRemoveVfioDevice(device *config.VFIODev) error { + cl := clh.client() + ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second) + defer cancel() + + _, err := cl.VmRemoveDevicePut(ctx, chclient.VmRemoveDevice{Id: device.ID}) + + if err != nil { + err = fmt.Errorf("failed to hotplug remove vfio device %+v %s", device, openAPIClientError(err)) + } + + return err +} + func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { span, _ := clh.trace("hotplugRemoveDevice") defer span.Finish() @@ -505,6 +519,8 @@ func (clh *cloudHypervisor) hotplugRemoveDevice(devInfo interface{}, devType dev switch devType { case blockDev: return nil, clh.hotplugRemoveBlockDevice(devInfo.(*config.BlockDrive)) + case vfioDev: + return nil, clh.hotplugRemoveVfioDevice(devInfo.(*config.VFIODev)) default: clh.Logger().WithFields(log.Fields{"devInfo": devInfo, "deviceType": devType}).Error("hotplugRemoveDevice: unsupported device") From cc4f02e2b6518a82fdcc1fb21efb4574a6aafd01 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 18:37:56 +0200 Subject: [PATCH 012/124] rust-agent: Remove unused macros This addresses the following warnings: Compiling rustjail v0.1.0 (/home/ddd/go/src/github.com/kata-containers-2.0/src/agent/rustjail) warning: unused `#[macro_use]` import --> rustjail/src/lib.rs:15:1 | 15 | #[macro_use] | ^^^^^^^^^^^^ | = note: `#[warn(unused_imports)]` on by default warning: unused macro definition --> rustjail/src/lib.rs:38:1 | 38 | / macro_rules! sl { 39 | | () => { 40 | | slog_scope::logger().new(o!("subsystem" => "rustjail")) 41 | | }; 42 | | } | |_^ | = note: `#[warn(unused_macros)]` on by default Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/lib.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index e71b7c643f..3fe93a8c9d 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -15,7 +15,6 @@ #[macro_use] #[cfg(test)] extern crate serial_test; -#[macro_use] extern crate serde; extern crate serde_json; #[macro_use] @@ -37,13 +36,6 @@ extern crate oci; extern crate path_absolutize; extern crate regex; -// Convenience macro to obtain the scope logger -macro_rules! sl { - () => { - slog_scope::logger().new(o!("subsystem" => "rustjail")) - }; -} - pub mod capabilities; pub mod cgroups; pub mod container; From 8ed61b1bb938c6f7f057a99a7164bf2df9e8dae8 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 24 Sep 2020 13:04:29 +0200 Subject: [PATCH 013/124] rust-agent: Remove useless braces This addresses the following warning: warning: unnecessary braces around assigned value --> src/rpc.rs:1411:26 | 1411 | detail.init_daemon = { unistd::getpid() == Pid::from_raw(1) }; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: remove these braces | = note: `#[warn(unused_braces)]` on by default Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index c1351e6360..2b3ed22077 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -1447,7 +1447,7 @@ fn get_agent_details() -> AgentDetails { detail.set_version(AGENT_VERSION.to_string()); detail.set_supports_seccomp(false); - detail.init_daemon = { unistd::getpid() == Pid::from_raw(1) }; + detail.init_daemon = unistd::getpid() == Pid::from_raw(1); detail.device_handlers = RepeatedField::new(); detail.storage_handlers = RepeatedField::from_vec( From e0b79eb57f366c33130c9482b63a642a40403d8d Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 18:29:49 +0200 Subject: [PATCH 014/124] rust-agent: Remove unused functions Fixes the following warning: Compiling logging v0.1.0 (/home/ddd/go/src/github.com/kata-containers-2.0/pkg/logging) warning: associated function is never used: `set_level` --> /home/ddd/go/src/github.com/kata-containers-2.0/pkg/logging/src/lib.rs:186:8 | 186 | fn set_level(&self, level: slog::Level) { | ^^^^^^^^^ | = note: `#[warn(dead_code)]` on by default Fixes: #750 Signed-off-by: Christophe de Dinechin --- pkg/logging/src/lib.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/logging/src/lib.rs b/pkg/logging/src/lib.rs index d22fd59e05..6708529643 100644 --- a/pkg/logging/src/lib.rs +++ b/pkg/logging/src/lib.rs @@ -182,12 +182,6 @@ impl RuntimeLevelFilter { level: Mutex::new(level), } } - - fn set_level(&self, level: slog::Level) { - let mut log_level = self.level.lock().unwrap(); - - *log_level = level; - } } impl Drain for RuntimeLevelFilter From 7d303ec2d027b8b3d5a20d95f7d2db775f1f7198 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 24 Sep 2020 13:08:20 +0200 Subject: [PATCH 015/124] rust-agent: Remove or rename unused variables Remove variables that are simply not used. Rename as _ variables where only initialization matters. This addresses the following warnings: warning: unused variable: `writer` --> src/main.rs:130:9 | 130 | let writer = unsafe { File::from_raw_fd(wfd) }; | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_writer` | = note: `#[warn(unused_variables)]` on by default warning: unused variable: `ctx` --> src/rpc.rs:782:9 | 782 | ctx: &ttrpc::TtrpcContext, | ^^^ help: if this is intentional, prefix it with an underscore: `_ctx` warning: unused variable: `ctx` --> src/rpc.rs:808:9 | 808 | ctx: &ttrpc::TtrpcContext, | ^^^ help: if this is intentional, prefix it with an underscore: `_ctx` warning: unused variable: `dns_list` --> src/rpc.rs:1152:16 | 1152 | Ok(dns_list) => { | ^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_dns_list` warning: value assigned to `child_stdin` is never read --> rustjail/src/container.rs:807:13 | 807 | let mut child_stdin = std::process::Stdio::null(); | ^^^^^^^^^^^^^^^ | = note: `#[warn(unused_assignments)]` on by default = help: maybe it is overwritten before being read? warning: value assigned to `child_stdout` is never read --> rustjail/src/container.rs:808:13 | 808 | let mut child_stdout = std::process::Stdio::null(); | ^^^^^^^^^^^^^^^^ | = help: maybe it is overwritten before being read? warning: value assigned to `child_stderr` is never read --> rustjail/src/container.rs:809:13 | 809 | let mut child_stderr = std::process::Stdio::null(); | ^^^^^^^^^^^^^^^^ | = help: maybe it is overwritten before being read? warning: value assigned to `stdin` is never read --> rustjail/src/container.rs:810:13 | 810 | let mut stdin = -1; | ^^^^^^^^^ | = help: maybe it is overwritten before being read? warning: value assigned to `stdout` is never read --> rustjail/src/container.rs:811:13 | 811 | let mut stdout = -1; | ^^^^^^^^^^ | = help: maybe it is overwritten before being read? warning: value assigned to `stderr` is never read --> rustjail/src/container.rs:812:13 | 812 | let mut stderr = -1; | ^^^^^^^^^^ | = help: maybe it is overwritten before being read? Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/container.rs | 29 +++++++++++++---------------- src/agent/rustjail/src/mount.rs | 16 ++++++++++------ src/agent/rustjail/src/process.rs | 4 ++-- src/agent/src/main.rs | 1 - src/agent/src/mount.rs | 2 +- src/agent/src/rpc.rs | 6 +++--- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 35ddb71e39..86745207cd 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -799,26 +799,23 @@ impl BaseContainer for LinuxContainer { unistd::close(pwfd); }); - let mut child_stdin = std::process::Stdio::null(); - let mut child_stdout = std::process::Stdio::null(); - let mut child_stderr = std::process::Stdio::null(); - let mut stdin = -1; - let mut stdout = -1; - let mut stderr = -1; + let mut child_stdin: std::process::Stdio; + let mut child_stdout: std::process::Stdio; + let mut child_stderr: std::process::Stdio; if tty { - let pseduo = pty::openpty(None, None)?; - p.term_master = Some(pseduo.master); - fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + let pseudo = pty::openpty(None, None)?; + p.term_master = Some(pseudo.master); + fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - child_stdin = unsafe { std::process::Stdio::from_raw_fd(pseduo.slave) }; - child_stdout = unsafe { std::process::Stdio::from_raw_fd(pseduo.slave) }; - child_stderr = unsafe { std::process::Stdio::from_raw_fd(pseduo.slave) }; + child_stdin = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; + child_stdout = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; + child_stderr = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; } else { - stdin = p.stdin.unwrap(); - stdout = p.stdout.unwrap(); - stderr = p.stderr.unwrap(); + let stdin = p.stdin.unwrap(); + let stdout = p.stdout.unwrap(); + let stderr = p.stderr.unwrap(); child_stdin = unsafe { std::process::Stdio::from_raw_fd(stdin) }; child_stdout = unsafe { std::process::Stdio::from_raw_fd(stdout) }; child_stderr = unsafe { std::process::Stdio::from_raw_fd(stderr) }; diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 34d4ae1889..1b4d24d15e 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -111,6 +111,7 @@ lazy_static! { } #[inline(always)] +#[allow(unused_variables)] fn mount( source: Option<&P1>, target: &P2, @@ -125,6 +126,7 @@ fn mount( target: &P, flags: MntFlags, @@ -421,6 +423,7 @@ fn mount_cgroups( Ok(()) } +#[allow(unused_variables)] fn pivot_root( new_root: &P1, put_old: &P2, @@ -553,6 +556,7 @@ fn parse_mount_table() -> Result> { } #[inline(always)] +#[allow(unused_variables)] fn chroot(path: &P) -> Result<(), nix::Error> { #[cfg(not(test))] return unistd::chroot(path); @@ -1004,8 +1008,8 @@ mod tests { // there is no spec.mounts, but should pass let ret = init_rootfs(stdout_fd, &spec, &cpath, &mounts, true); assert!(ret.is_ok(), "Should pass. Got: {:?}", ret); - let ret = fs::remove_dir_all(rootfs.path().join("dev")); - let ret = fs::create_dir(rootfs.path().join("dev")); + let _ = fs::remove_dir_all(rootfs.path().join("dev")); + let _ = fs::create_dir(rootfs.path().join("dev")); // Adding bad mount point to spec.mounts spec.mounts.push(oci::Mount { @@ -1023,8 +1027,8 @@ mod tests { ret ); spec.mounts.pop(); - let ret = fs::remove_dir_all(rootfs.path().join("dev")); - let ret = fs::create_dir(rootfs.path().join("dev")); + let _ = fs::remove_dir_all(rootfs.path().join("dev")); + let _ = fs::create_dir(rootfs.path().join("dev")); // mounting a cgroup spec.mounts.push(oci::Mount { @@ -1037,8 +1041,8 @@ mod tests { let ret = init_rootfs(stdout_fd, &spec, &cpath, &mounts, true); assert!(ret.is_ok(), "Should pass. Got: {:?}", ret); spec.mounts.pop(); - let ret = fs::remove_dir_all(rootfs.path().join("dev")); - let ret = fs::create_dir(rootfs.path().join("dev")); + let _ = fs::remove_dir_all(rootfs.path().join("dev")); + let _ = fs::create_dir(rootfs.path().join("dev")); // mounting /dev spec.mounts.push(oci::Mount { diff --git a/src/agent/rustjail/src/process.rs b/src/agent/rustjail/src/process.rs index 665bd8d071..c832c03c1b 100644 --- a/src/agent/rustjail/src/process.rs +++ b/src/agent/rustjail/src/process.rs @@ -151,11 +151,11 @@ mod tests { #[test] fn test_create_extended_pipe() { // Test the default - let (r, w) = create_extended_pipe(OFlag::O_CLOEXEC, 0).unwrap(); + let (_r, _w) = create_extended_pipe(OFlag::O_CLOEXEC, 0).unwrap(); // Test setting to the max size let max_size = get_pipe_max_size(); - let (r, w) = create_extended_pipe(OFlag::O_CLOEXEC, max_size).unwrap(); + let (_, w) = create_extended_pipe(OFlag::O_CLOEXEC, max_size).unwrap(); let actual_size = get_pipe_size(w); assert_eq!(max_size, actual_size); } diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 946adefaa9..e75f55fa04 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -129,7 +129,6 @@ fn main() -> Result<()> { // support vsock log let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?; - let writer = unsafe { File::from_raw_fd(wfd) }; let agentConfig = AGENT_CONFIG.clone(); diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 2d8ade82bc..a85d5c3a55 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -1088,7 +1088,7 @@ mod tests { #[test] fn test_get_cgroup_v2_mounts() { - let dir = tempdir().expect("failed to create tmpdir"); + let _ = tempdir().expect("failed to create tmpdir"); let drain = slog::Discard; let logger = slog::Logger::root(drain, o!()); let result = get_cgroup_mounts(&logger, "", true); diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 2b3ed22077..a62fe44265 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -790,7 +790,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { fn pause_container( &self, - ctx: &ttrpc::TtrpcContext, + _ctx: &ttrpc::TtrpcContext, req: protocols::agent::PauseContainerRequest, ) -> ttrpc::Result { let cid = req.get_container_id(); @@ -816,7 +816,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { fn resume_container( &self, - ctx: &ttrpc::TtrpcContext, + _ctx: &ttrpc::TtrpcContext, req: protocols::agent::ResumeContainerRequest, ) -> ttrpc::Result { let cid = req.get_container_id(); @@ -1160,7 +1160,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { }; match setup_guest_dns(sl!(), req.dns.to_vec()) { - Ok(dns_list) => { + Ok(_) => { let sandbox = self.sandbox.clone(); let mut s = sandbox.lock().unwrap(); let _ = req From 76298c12b7699ef2ee098449487e3d55d85c70a7 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 18:55:22 +0200 Subject: [PATCH 016/124] rust-agent: Remove or rename unused parameters Parameters that are never used were removed. Parameters that are unused, but necessary because of some common interface were renamed with a _ prefix. In one case, consume the parameter by adding an info! call, and fix a minor typo in a message in the same function. This addresses the following warning: warning: unused variable: `child` --> rustjail/src/container.rs:1128:5 | 1128 | child: &mut Child, | ^^^^^ help: if this is intentional, prefix it with an underscore: `_child` warning: unused variable: `logger` --> rustjail/src/container.rs:1049:22 | 1049 | fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> Result<()> { | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_logger` Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/cgroups/fs/mod.rs | 6 +++--- src/agent/rustjail/src/container.rs | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 1685ac4a33..13a4c7aa20 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -230,7 +230,7 @@ impl CgroupManager for Manager { } fn set_network_resources( - cg: &cgroups::Cgroup, + _cg: &cgroups::Cgroup, network: &LinuxNetwork, res: &mut cgroups::Resources, ) -> Result<()> { @@ -259,7 +259,7 @@ fn set_network_resources( } fn set_devices_resources( - cg: &cgroups::Cgroup, + _cg: &cgroups::Cgroup, device_resources: &Vec, res: &mut cgroups::Resources, ) -> Result<()> { @@ -288,7 +288,7 @@ fn set_devices_resources( } fn set_hugepages_resources( - cg: &cgroups::Cgroup, + _cg: &cgroups::Cgroup, hugepage_limits: &Vec, res: &mut cgroups::Resources, ) -> Result<()> { diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 86745207cd..7f3b4d6b08 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -888,7 +888,6 @@ impl BaseContainer for LinuxContainer { &p, self.cgroup_manager.as_ref().unwrap(), &st, - &mut child, pwfd, prfd, ) { @@ -1039,8 +1038,9 @@ fn do_exec(args: &[String]) -> ! { } fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> Result<()> { + info!(logger, "updating namespaces"); let linux = match spec.linux.as_mut() { - None => return Err(anyhow!("Spec didn't container linux field")), + None => return Err(anyhow!("Spec didn't contain linux field")), Some(l) => l, }; @@ -1117,7 +1117,6 @@ fn join_namespaces( p: &Process, cm: &FsManager, st: &OCIState, - _child: &mut Child, pwfd: RawFd, prfd: RawFd, ) -> Result<()> { From 8d8adb6887ed22ab7dde767079300b6b18013842 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 18:45:06 +0200 Subject: [PATCH 017/124] rust-agent: Remove uses of deprecated functions This addresses the following: warning: use of deprecated item 'std::error::Error::description': use the Display impl or to_string() --> rustjail/src/container.rs:1598:31 | 1598 | ... e.description(), | ^^^^^^^^^^^ | = note: `#[warn(deprecated)]` on by default Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/container.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 7f3b4d6b08..68a1c13b40 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -1545,7 +1545,7 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { info!( logger, "wait child error: {} {}", - e.description(), + e, e.raw_os_error().unwrap() ); From 86bc15178752968ad860dbef16dc9d37c173cccd Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 19:08:06 +0200 Subject: [PATCH 018/124] rust-agent: Remove 'mut' where not needed Addresses the following warning (and a few similar ones): warning: variable does not need to be mutable --> rustjail/src/container.rs:369:9 | 369 | let mut oci_process: oci::Process = serde_json::from_str(process_str)?; | ----^^^^^^^^^^^ | | | help: remove this `mut` | = note: `#[warn(unused_mut)]` on by default Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/cgroups/fs/mod.rs | 4 ++-- src/agent/rustjail/src/container.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 13a4c7aa20..0999fd5f43 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -468,7 +468,7 @@ fn build_blk_io_device_throttle_resource( fn linux_device_to_cgroup_device(d: &LinuxDevice) -> DeviceResource { let dev_type = DeviceType::from_char(d.r#type.chars().next()).unwrap(); - let mut permissions = vec![ + let permissions = vec![ DevicePermissions::Read, DevicePermissions::Write, DevicePermissions::MkNod, @@ -518,7 +518,7 @@ fn lines_to_map(content: &str) -> HashMap { .lines() .map(|x| x.split_whitespace().collect::>()) .filter(|x| x.len() == 2 && x[1].parse::().is_ok()) - .fold(HashMap::new(), |mut hm, mut x| { + .fold(HashMap::new(), |mut hm, x| { hm.insert(x[0].to_string(), x[1].parse::().unwrap()); hm }) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 68a1c13b40..3463fbcc75 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -364,7 +364,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { let buf = read_sync(crfd)?; let process_str = std::str::from_utf8(&buf)?; - let mut oci_process: oci::Process = serde_json::from_str(process_str)?; + let oci_process: oci::Process = serde_json::from_str(process_str)?; log_child!(cfd_log, "notify parent to send cgroup manager"); write_sync(cwfd, SYNC_SUCCESS, "")?; @@ -799,9 +799,9 @@ impl BaseContainer for LinuxContainer { unistd::close(pwfd); }); - let mut child_stdin: std::process::Stdio; - let mut child_stdout: std::process::Stdio; - let mut child_stderr: std::process::Stdio; + let child_stdin: std::process::Stdio; + let child_stdout: std::process::Stdio; + let child_stderr: std::process::Stdio; if tty { let pseudo = pty::openpty(None, None)?; @@ -865,7 +865,7 @@ impl BaseContainer for LinuxContainer { child = child.env(FIFO_FD, format!("{}", fifofd)); } - let mut child = child.spawn()?; + let child = child.spawn()?; unistd::close(crfd)?; unistd::close(cwfd)?; From 19cb65729909c09b99c3258d791dc25db278aacb Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 19:17:17 +0200 Subject: [PATCH 019/124] rust-agent: Remove unused code that has undefined behavior Some functions have undefined behavior and are not actually used. This addresses the following warning: warning: the type `oci::User` does not permit zero-initialization --> rustjail/src/lib.rs:99:18 | 99 | unsafe { MaybeUninit::zeroed().assume_init() } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | | | this code causes undefined behavior when executed | help: use `MaybeUninit` instead, and only call `assume_init` after initialization is done | = note: `#[warn(invalid_value)]` on by default note: `std::ptr::Unique` must be non-null (in this struct field) warning: the type `protocols::oci::Process` does not permit zero-initialization --> rustjail/src/lib.rs:146:14 | 146 | unsafe { MaybeUninit::zeroed().assume_init() } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | | | this code causes undefined behavior when executed | help: use `MaybeUninit` instead, and only call `assume_init` after initialization is done | note: `std::ptr::Unique` must be non-null (in this struct field) Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/lib.rs | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index 3fe93a8c9d..d044408474 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -69,7 +69,6 @@ use protocols::oci::{ Root as grpcRoot, Spec as grpcSpec, }; use std::collections::HashMap; -use std::mem::MaybeUninit; pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess { let console_size = if p.ConsoleSize.is_some() { @@ -91,7 +90,12 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess { username: u.Username.clone(), } } else { - unsafe { MaybeUninit::zeroed().assume_init() } + ociUser { + uid: 0, + gid: 0, + additional_gids: vec![], + username: String::from(""), + } }; let capabilities = if p.Capabilities.is_some() { @@ -136,11 +140,6 @@ pub fn process_grpc_to_oci(p: &grpcProcess) -> ociProcess { } } -fn process_oci_to_grpc(_p: ociProcess) -> grpcProcess { - // dont implement it for now - unsafe { MaybeUninit::zeroed().assume_init() } -} - fn root_grpc_to_oci(root: &grpcRoot) -> ociRoot { ociRoot { path: root.Path.clone(), @@ -148,10 +147,6 @@ fn root_grpc_to_oci(root: &grpcRoot) -> ociRoot { } } -fn root_oci_to_grpc(_root: &ociRoot) -> grpcRoot { - unsafe { MaybeUninit::zeroed().assume_init() } -} - fn mount_grpc_to_oci(m: &grpcMount) -> ociMount { ociMount { destination: m.destination.clone(), @@ -161,10 +156,6 @@ fn mount_grpc_to_oci(m: &grpcMount) -> ociMount { } } -fn mount_oci_to_grpc(_m: &ociMount) -> grpcMount { - unsafe { MaybeUninit::zeroed().assume_init() } -} - use oci::Hook as ociHook; use protocols::oci::Hook as grpcHook; @@ -195,10 +186,6 @@ fn hooks_grpc_to_oci(h: &grpcHooks) -> ociHooks { } } -fn hooks_oci_to_grpc(_h: &ociHooks) -> grpcHooks { - unsafe { MaybeUninit::zeroed().assume_init() } -} - use oci::{ LinuxDevice as ociLinuxDevice, LinuxIDMapping as ociLinuxIDMapping, LinuxIntelRdt as ociLinuxIntelRdt, LinuxNamespace as ociLinuxNamespace, @@ -565,10 +552,6 @@ pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec { } } -pub fn oci_to_grpc(_oci: &ociSpec) -> grpcSpec { - unsafe { MaybeUninit::zeroed().assume_init() } -} - #[cfg(test)] mod tests { #[test] From 4db3f9e226953033de47b6d9c835cf7e9eb3519a Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 19:47:52 +0200 Subject: [PATCH 020/124] rust-agent: Ignore write errors while writing to the logs When we are writing to the logs and there is an error doing so, there is not much we can do. Chances are that a panic would make things worse. So let it go through. warning: unused `std::result::Result` that must be used --> rustjail/src/sync.rs:26:9 | 26 | write_count(lfd, log_str.as_bytes(), log_str.len()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | ::: rustjail/src/container.rs:339:13 | 339 | log_child!(cfd_log, "child exit: {:?}", e); | ------------------------------------------- in this macro invocation | = note: this `Result` may be an `Err` variant, which should be handled = note: this warning originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/sync.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent/rustjail/src/sync.rs b/src/agent/rustjail/src/sync.rs index 20bf7b4706..8ce43b270f 100644 --- a/src/agent/rustjail/src/sync.rs +++ b/src/agent/rustjail/src/sync.rs @@ -23,7 +23,8 @@ macro_rules! log_child { let lfd = $fd; let mut log_str = format_args!($($arg)+).to_string(); log_str.push('\n'); - write_count(lfd, log_str.as_bytes(), log_str.len()); + // Ignore error writing to the logger, not much we can do + let _ = write_count(lfd, log_str.as_bytes(), log_str.len()); }) } From 73ab9b1d6d3a5b7329ca40c1f40e8609702cde34 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 24 Sep 2020 13:45:58 +0200 Subject: [PATCH 021/124] rust-agent: Report errors to caller if possible Various recently added error-causing calls This addresses the following warning: warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:93:9 | 93 | cg.add_task(CgroupPid::from(pid as u64)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:196:17 | 196 | freezer_controller.thaw(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:199:17 | 199 | freezer_controller.freeze(); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:365:9 | 365 | cpuset_controller.set_cpus(&cpu.cpus); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:369:9 | 369 | cpuset_controller.set_mems(&cpu.mems); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:381:13 | 381 | cpu_controller.set_shares(shares); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:385:5 | 385 | cpu_controller.set_cfs_quota_and_period(cpu.quota, cpu.period); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/cgroups/fs/mod.rs:1061:13 | 1061 | cpuset_controller.set_cpus(cpuset_cpus); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled The specific case of cpu_controller.set_cfs_quota_and_period is addressed in a way that changes the logic following a suggestion by Liu Bin, who had just added the code. Fixes: #750 Suggested-by: Liu Bin Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/cgroups/fs/mod.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 0999fd5f43..8096bebe6b 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -91,7 +91,7 @@ impl CgroupManager for Manager { let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); let cg = load_or_create(h, &self.cpath); - cg.add_task(CgroupPid::from(pid as u64)); + cg.add_task(CgroupPid::from(pid as u64))?; Ok(()) } @@ -194,10 +194,10 @@ impl CgroupManager for Manager { let freezer_controller: &FreezerController = cg.controller_of().unwrap(); match state { FreezerState::Thawed => { - freezer_controller.thaw(); + freezer_controller.thaw()?; } FreezerState::Frozen => { - freezer_controller.freeze(); + freezer_controller.freeze()?; } _ => { return Err(nix::Error::Sys(Errno::EINVAL).into()); @@ -363,11 +363,11 @@ fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCPU) -> Result<()> { let cpuset_controller: &CpuSetController = cg.controller_of().unwrap(); if !cpu.cpus.is_empty() { - cpuset_controller.set_cpus(&cpu.cpus); + cpuset_controller.set_cpus(&cpu.cpus)?; } if !cpu.mems.is_empty() { - cpuset_controller.set_mems(&cpu.mems); + cpuset_controller.set_mems(&cpu.mems)?; } let cpu_controller: &CpuController = cg.controller_of().unwrap(); @@ -379,11 +379,12 @@ fn set_cpu_resources(cg: &cgroups::Cgroup, cpu: &LinuxCPU) -> Result<()> { shares }; if shares != 0 { - cpu_controller.set_shares(shares); + cpu_controller.set_shares(shares)?; } } - cpu_controller.set_cfs_quota_and_period(cpu.quota, cpu.period); + set_resource!(cpu_controller, set_cfs_quota, cpu, quota); + set_resource!(cpu_controller, set_cfs_period, cpu, period); set_resource!(cpu_controller, set_rt_runtime, cpu, realtime_runtime); set_resource!(cpu_controller, set_rt_period_us, cpu, realtime_period); @@ -1059,7 +1060,7 @@ impl Manager { info!(sl!(), "updating cpuset for path {:?}", &r_path); let cg = load_or_create(h, &r_path); let cpuset_controller: &CpuSetController = cg.controller_of().unwrap(); - cpuset_controller.set_cpus(cpuset_cpus); + cpuset_controller.set_cpus(cpuset_cpus)?; } Ok(()) From 9adb7b7c28d2a94679815c71d9942d6de9cb5392 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 17 Sep 2020 18:40:56 +0200 Subject: [PATCH 022/124] rust-agent: Remove unused imports This addresses the following warnings (and similar ones):: Compiling rustjail v0.1.0 (/home/ddd/go/src/github.com/kata-containers-2.0/src/agent/rustjail) warning: unused import: `debug` --> rustjail/src/container.rs:57:12 | 57 | use slog::{debug, info, o, Logger}; | ^^^^^ warning: unused imports: `AddressFamily`, `SockFlag`, `SockType`, `self` --> rustjail/src/process.rs:18:24 | 18 | use nix::sys::socket::{self, AddressFamily, SockFlag, SockType}; | ^^^^ ^^^^^^^^^^^^^ ^^^^^^^^ ^^^^^^^^ warning: unused import: `nix::Error` --> rustjail/src/process.rs:23:5 | 23 | use nix::Error; | ^^^^^^^^^^ warning: unused import: `protobuf::RepeatedField` --> rustjail/src/validator.rs:11:5 | 11 | use protobuf::RepeatedField; | ^^^^^^^^^^^^^^^^^^^^^^^ Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/cgroups/fs/mod.rs | 9 ++-- src/agent/rustjail/src/cgroups/mod.rs | 1 - src/agent/rustjail/src/container.rs | 7 ++- src/agent/rustjail/src/mount.rs | 4 +- src/agent/rustjail/src/process.rs | 2 - src/agent/rustjail/src/validator.rs | 1 - src/agent/src/main.rs | 3 -- src/agent/src/network.rs | 6 +-- src/agent/src/rpc.rs | 3 +- src/agent/src/sandbox.rs | 2 - tools/agent-ctl/Cargo.lock | 63 ++---------------------- tools/agent-ctl/src/utils.rs | 3 +- 12 files changed, 17 insertions(+), 87 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 8096bebe6b..1b70a28edf 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use cgroups::blkio::{BlkIo, BlkIoController, BlkIoData, IoService}; +use cgroups::blkio::{BlkIoController, BlkIoData, IoService}; use cgroups::cpu::CpuController; use cgroups::cpuacct::CpuAcctController; use cgroups::cpuset::CpuSetController; @@ -15,18 +15,18 @@ use cgroups::memory::MemController; use cgroups::pid::PidController; use cgroups::{ BlkIoDeviceResource, BlkIoDeviceThrottleResource, Cgroup, CgroupPid, Controller, - DeviceResource, DeviceResources, HugePageResource, MaxValue, NetworkPriority, + DeviceResource, HugePageResource, MaxValue, NetworkPriority, }; use crate::cgroups::Manager as CgroupManager; use crate::container::DEFAULT_DEVICES; -use anyhow::{anyhow, Context, Error, Result}; +use anyhow::{anyhow, Context, Result}; use lazy_static; use libc::{self, pid_t}; use nix::errno::Errno; use oci::{ LinuxBlockIO, LinuxCPU, LinuxDevice, LinuxDeviceCgroup, LinuxHugepageLimit, LinuxMemory, - LinuxNetwork, LinuxPids, LinuxResources, LinuxThrottleDevice, LinuxWeightDevice, + LinuxNetwork, LinuxPids, LinuxResources, }; use protobuf::{CachedSize, RepeatedField, SingularPtrField, UnknownFields}; @@ -34,7 +34,6 @@ use protocols::agent::{ BlkioStats, BlkioStatsEntry, CgroupStats, CpuStats, CpuUsage, HugetlbStats, MemoryData, MemoryStats, PidsStats, ThrottlingData, }; -use regex::Regex; use std::collections::HashMap; use std::fs; use std::path::Path; diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs index 0e6052542a..c99ef469ac 100644 --- a/src/agent/rustjail/src/cgroups/mod.rs +++ b/src/agent/rustjail/src/cgroups/mod.rs @@ -7,7 +7,6 @@ use anyhow::{anyhow, Result}; use oci::LinuxResources; use protocols::agent::CgroupStats; -use std::collections::HashMap; use cgroups::freezer::FreezerState; diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 3463fbcc75..d76564d0c1 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -19,7 +19,7 @@ use libc::pid_t; use oci::{LinuxDevice, LinuxIDMapping}; use std::clone::Clone; use std::fmt::Display; -use std::process::{Child, Command}; +use std::process::Command; use cgroups::freezer::FreezerState; @@ -30,7 +30,7 @@ use crate::specconv::CreateOpts; use crate::sync::*; // use crate::stats::Stats; use crate::capabilities::{self, CAPSMAP}; -use crate::cgroups::fs::{self as fscgroup, Manager as FsManager}; +use crate::cgroups::fs::Manager as FsManager; use crate::cgroups::Manager; use crate::{mount, validator}; @@ -55,7 +55,7 @@ use std::io::BufRead; use std::io::BufReader; use std::os::unix::io::FromRawFd; -use slog::{debug, info, o, Logger}; +use slog::{info, o, Logger}; const STATE_FILENAME: &'static str = "state.json"; const EXEC_FIFO_FILENAME: &'static str = "exec.fifo"; @@ -1424,7 +1424,6 @@ fn set_sysctls(sysctls: &HashMap) -> Result<()> { Ok(()) } -use std::error::Error as StdError; use std::io::Read; use std::os::unix::process::ExitStatusExt; use std::process::Stdio; diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 1b4d24d15e..0e82c4aee8 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -7,7 +7,9 @@ use anyhow::{anyhow, bail, Context, Error, Result}; use libc::uid_t; use nix::errno::Errno; use nix::fcntl::{self, OFlag}; -use nix::mount::{self, MntFlags, MsFlags}; +#[cfg(not(test))] +use nix::mount; +use nix::mount::{MntFlags, MsFlags}; use nix::sys::stat::{self, Mode, SFlag}; use nix::unistd::{self, Gid, Uid}; use nix::NixPath; diff --git a/src/agent/rustjail/src/process.rs b/src/agent/rustjail/src/process.rs index c832c03c1b..f27c4cda02 100644 --- a/src/agent/rustjail/src/process.rs +++ b/src/agent/rustjail/src/process.rs @@ -15,12 +15,10 @@ use std::sync::mpsc::Sender; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use nix::sys::signal::{self, Signal}; -use nix::sys::socket::{self, AddressFamily, SockFlag, SockType}; use nix::sys::wait::{self, WaitStatus}; use nix::unistd::{self, Pid}; use nix::Result; -use nix::Error; use oci::Process as OCIProcess; use slog::Logger; diff --git a/src/agent/rustjail/src/validator.rs b/src/agent/rustjail/src/validator.rs index 14ffef1bd8..deaf7c14ac 100644 --- a/src/agent/rustjail/src/validator.rs +++ b/src/agent/rustjail/src/validator.rs @@ -8,7 +8,6 @@ use anyhow::{anyhow, Result}; use lazy_static; use nix::errno::Errno; use oci::{LinuxIDMapping, LinuxNamespace, Spec}; -use protobuf::RepeatedField; use std::collections::HashMap; use std::path::{Component, PathBuf}; diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index e75f55fa04..0dc8667f84 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -25,7 +25,6 @@ extern crate scopeguard; #[macro_use] extern crate slog; -#[macro_use] extern crate netlink; use crate::netlink::{RtnlHandle, NETLINK_ROUTE}; @@ -637,8 +636,6 @@ fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Re #[cfg(test)] mod tests { use super::*; - use std::fs::File; - use std::io::Write; use tempfile::tempdir; #[test] diff --git a/src/agent/src/network.rs b/src/agent/src/network.rs index 01a088dac9..1fccf5eeec 100644 --- a/src/agent/src/network.rs +++ b/src/agent/src/network.rs @@ -3,15 +3,13 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, Context, Result}; -use nix::mount::{self, MntFlags, MsFlags}; +use anyhow::{anyhow, Result}; +use nix::mount::{self, MsFlags}; use protocols::types::{Interface, Route}; use slog::Logger; use std::collections::HashMap; use std::fs; -use crate::Sandbox; - const KATA_GUEST_SANDBOX_DNS_FILE: &str = "/run/kata-containers/sandbox/resolv.conf"; const GUEST_DNS_FILE: &str = "/etc/resolv.conf"; diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index a62fe44265..ed71402742 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -4,7 +4,7 @@ // use std::path::Path; -use std::sync::mpsc::{channel, Sender}; +use std::sync::mpsc::channel; use std::sync::{Arc, Mutex}; use ttrpc; @@ -40,7 +40,6 @@ use crate::metrics::get_metrics; use crate::mount::{add_storages, remove_mounts, BareMount, STORAGEHANDLERLIST}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::network::setup_guest_dns; -use crate::network::Network; use crate::random; use crate::sandbox::Sandbox; use crate::version::{AGENT_VERSION, API_VERSION}; diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 8c5eacbe49..f871d3ef72 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -7,10 +7,8 @@ use crate::linux_abi::*; use crate::mount::{get_mount_fs_type, remove_mounts, TYPEROOTFS}; use crate::namespace::Namespace; -use crate::namespace::NSTYPEPID; use crate::network::Network; use anyhow::{anyhow, Context, Result}; -use cgroups; use libc::pid_t; use netlink::{RtnlHandle, NETLINK_ROUTE}; use oci::{Hook, Hooks}; diff --git a/tools/agent-ctl/Cargo.lock b/tools/agent-ctl/Cargo.lock index 276379c065..ea06728c34 100644 --- a/tools/agent-ctl/Cargo.lock +++ b/tools/agent-ctl/Cargo.lock @@ -1,20 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "addr2line" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602d785912f476e480434627e8732e6766b760c045bbf897d9dfaa9f4fbd399c" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler32" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" - [[package]] name = "aho-corasick" version = "0.7.13" @@ -35,9 +20,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" +checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b" [[package]] name = "arc-swap" @@ -74,20 +59,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -[[package]] -name = "backtrace" -version = "0.3.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05100821de9e028f12ae3d189176b41ee198341eb8f369956407fea2f5cc666c" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - [[package]] name = "base64" version = "0.11.0" @@ -240,7 +211,6 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" dependencies = [ - "backtrace", "version_check", ] @@ -267,12 +237,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" - [[package]] name = "hermit-abi" version = "0.1.14" @@ -361,15 +325,6 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" -[[package]] -name = "miniz_oxide" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" -dependencies = [ - "adler32", -] - [[package]] name = "nix" version = "0.16.1" @@ -415,12 +370,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" - [[package]] name = "oci" version = "0.1.0" @@ -606,19 +555,13 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustjail" version = "0.1.0" dependencies = [ + "anyhow", "caps", "dirs", - "error-chain", "lazy_static", "libc", "nix 0.17.0", diff --git a/tools/agent-ctl/src/utils.rs b/tools/agent-ctl/src/utils.rs index 6864847605..7dff5d8cb6 100644 --- a/tools/agent-ctl/src/utils.rs +++ b/tools/agent-ctl/src/utils.rs @@ -8,8 +8,7 @@ use anyhow::{anyhow, Result}; use oci::{Process as ociProcess, Root as ociRoot, Spec as ociSpec}; use protocols::oci::{ Box as grpcBox, Linux as grpcLinux, LinuxCapabilities as grpcLinuxCapabilities, - POSIXRlimit as grpcPOSIXRlimit, Process as grpcProcess, Root as grpcRoot, Spec as grpcSpec, - User as grpcUser, + Process as grpcProcess, Root as grpcRoot, Spec as grpcSpec, User as grpcUser, }; use rand::Rng; use slog::{debug, warn}; From ce54e5dd571724a03e44703ef52350746f147a1f Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 2 Oct 2020 15:35:34 +0200 Subject: [PATCH 023/124] rust-agent: Log returned errors rather than ignore them In a number of cases, we have functions that return a Result<...> and where the possible error case is simply ignored. This is a bit unhealthy. Add a `check!` macro that allows us to not ignore error values that we want to log, while not interrupting the flow by returning them. This is useful for low-level functions such as `signal::kill` or `unistd::close` where an error is probably significant, but should not necessarily interrupt the flow of the program (i.e. using `call()?` is not the right answer. The check! macro is then used on low-level calls. This addresses the following warnings from #750: This addresses the following warning: warning: unused `std::result::Result` that must be used --> /home/ddd/go/src/github.com/kata-containers-2.0/src/agent/rustjail/src/container.rs:903:17 | 903 | signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> /home/ddd/go/src/github.com/kata-containers-2.0/src/agent/rustjail/src/container.rs:916:17 | 916 | signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:340:13 | 340 | write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:554:13 | 554 | / write_sync( 555 | | cwfd, 556 | | SYNC_FAILED, 557 | | format!("setgroups failed: {:?}", e).as_str(), 558 | | ); | |______________^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:340:13 | 340 | write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:340:13 | 340 | write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:554:13 | 554 | / write_sync( 555 | | cwfd, 556 | | SYNC_FAILED, 557 | | format!("setgroups failed: {:?}", e).as_str(), 558 | | ); | |______________^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:626:5 | 626 | unistd::close(cfd_log); | ^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:627:5 | 627 | unistd::close(crfd); | ^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:628:5 | 628 | unistd::close(cwfd); | ^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:770:9 | 770 | fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:799:9 | 799 | fcntl::fcntl(prfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:800:9 | 800 | fcntl::fcntl(pwfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:803:13 | 803 | unistd::close(prfd); | ^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:930:9 | 930 | log_handler.join(); | ^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:803:13 | 803 | unistd::close(prfd); | ^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:804:13 | 804 | unistd::close(pwfd); | ^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:842:13 | 842 | sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/container.rs:843:13 | 843 | unistd::close(old_pid_ns); | ^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled Fixes: #844 Fixes: #750 Suggested-by: Tim Zhang Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/container.rs | 93 ++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index d76564d0c1..e30f4d420a 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -70,6 +70,17 @@ const CLOG_FD: &str = "CLOG_FD"; const FIFO_FD: &str = "FIFO_FD"; const HOME_ENV_KEY: &str = "HOME"; +#[macro_export] +macro_rules! check { + ($what:expr, $where:expr) => ({ + if let Err(e) = $what { + let subsystem = $where; + let logger = slog_scope::logger().new(o!("subsystem" => subsystem)); + warn!(logger, "{:?}", e); + } + }) +} + #[derive(PartialEq, Clone, Copy)] pub enum Status { CREATED, @@ -336,7 +347,10 @@ pub fn init_child() { Ok(_) => (), Err(e) => { log_child!(cfd_log, "child exit: {:?}", e); - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); + check!( + write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), + "write_sync in init_child()" + ); return; } } @@ -471,11 +485,17 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { if let Err(e) = sched::setns(fd, s) { if s == CloneFlags::CLONE_NEWUSER { if e.as_errno().unwrap() != Errno::EINVAL { - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); + check!( + write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), + "write_sync for CLONE_NEWUSER" + ); return Err(e.into()); } } else { - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); + check!( + write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), + "write_sync for sched::setns" + ); return Err(e.into()); } } @@ -550,10 +570,13 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { if guser.additional_gids.len() > 0 { setgroups(guser.additional_gids.as_slice()).map_err(|e| { - write_sync( - cwfd, - SYNC_FAILED, - format!("setgroups failed: {:?}", e).as_str(), + check!( + write_sync( + cwfd, + SYNC_FAILED, + format!("setgroups failed: {:?}", e).as_str() + ), + "write_sync for setgroups" ); e })?; @@ -622,9 +645,9 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { // notify parent that the child's ready to start write_sync(cwfd, SYNC_SUCCESS, "")?; log_child!(cfd_log, "ready to run exec"); - unistd::close(cfd_log); - unistd::close(crfd); - unistd::close(cwfd); + check!(unistd::close(cfd_log), "closing cfd log"); + check!(unistd::close(crfd), "closing crfd"); + check!(unistd::close(cwfd), "closing cwfd"); if oci_process.terminal { unistd::setsid()?; @@ -762,7 +785,10 @@ impl BaseContainer for LinuxContainer { let st = self.oci_state()?; let (pfd_log, cfd_log) = unistd::pipe().context("failed to create pipe")?; - fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + check!( + fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), + "fcntl pfd log FD_CLOEXEC" + ); let child_logger = logger.new(o!("action" => "child process log")); let log_handler = thread::spawn(move || { @@ -791,12 +817,18 @@ impl BaseContainer for LinuxContainer { info!(logger, "exec fifo opened!"); let (prfd, cwfd) = unistd::pipe().context("failed to create pipe")?; let (crfd, pwfd) = unistd::pipe().context("failed to create pipe")?; - fcntl::fcntl(prfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - fcntl::fcntl(pwfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + check!( + fcntl::fcntl(prfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), + "fcntl prfd FD_CLOEXEC" + ); + check!( + fcntl::fcntl(pwfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), + "fcntl pwfd FD_COLEXEC" + ); defer!({ - unistd::close(prfd); - unistd::close(pwfd); + check!(unistd::close(prfd), "close prfd"); + check!(unistd::close(pwfd), "close pwfd"); }); let child_stdin: std::process::Stdio; @@ -806,8 +838,14 @@ impl BaseContainer for LinuxContainer { if tty { let pseudo = pty::openpty(None, None)?; p.term_master = Some(pseudo.master); - fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + check!( + fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), + "fnctl pseudo.master" + ); + check!( + fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), + "fcntl pseudo.slave" + ); child_stdin = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; child_stdout = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; @@ -834,8 +872,11 @@ impl BaseContainer for LinuxContainer { //restore the parent's process's pid namespace. defer!({ - sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID); - unistd::close(old_pid_ns); + check!( + sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID), + "settns CLONE_NEWPID" + ); + check!(unistd::close(old_pid_ns), "close old pid namespace"); }); let pidns = get_pid_namespace(&self.logger, linux)?; @@ -877,7 +918,7 @@ impl BaseContainer for LinuxContainer { } if p.init { - unistd::close(fifofd); + check!(unistd::close(fifofd), "close fifofd"); } info!(logger, "child pid: {}", p.pid); @@ -895,7 +936,10 @@ impl BaseContainer for LinuxContainer { Err(e) => { error!(logger, "create container process error {:?}", e); // kill the child process. - signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)); + check!( + signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)), + "signal::kill joining namespaces" + ); return Err(e); } }; @@ -908,7 +952,10 @@ impl BaseContainer for LinuxContainer { let (exit_pipe_r, exit_pipe_w) = unistd::pipe2(OFlag::O_CLOEXEC) .context("failed to create pipe") .map_err(|e| { - signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL)); + check!( + signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL)), + "signal::kill creating pipe" + ); e })?; @@ -922,7 +969,7 @@ impl BaseContainer for LinuxContainer { self.processes.insert(p.pid, p); info!(logger, "wait on child log handler"); - log_handler.join(); + check!(log_handler.join(), "joining log handler"); info!(logger, "create process completed"); return Ok(()); } From 8f7a4842c282050a178902d20810dd58c5c790b2 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Tue, 6 Oct 2020 15:00:17 +0200 Subject: [PATCH 024/124] rust-agent: Identify unused results in tests Assign unused results to _ in order to silence warnings. This addresses the following warnings: warning: unused `std::result::Result` that must be used --> rustjail/src/mount.rs:1182:16 | 1182 | defer!(unistd::chdir(&olddir);); | ^^^^^^^^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_must_use)]` on by default = note: this `Result` may be an `Err` variant, which should be handled warning: unused `std::result::Result` that must be used --> rustjail/src/mount.rs:1183:9 | 1183 | unistd::chdir(tempdir.path()); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | = note: this `Result` may be an `Err` variant, which should be handled While in regular code, we want to log possible errors, in test code it's OK to simply ignore the returned value. Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/rustjail/src/mount.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 0e82c4aee8..4256a9832d 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -1185,8 +1185,8 @@ mod tests { let tempdir = tempdir().unwrap(); let olddir = unistd::getcwd().unwrap(); - defer!(unistd::chdir(&olddir);); - unistd::chdir(tempdir.path()); + defer!(let _ = unistd::chdir(&olddir);); + let _ = unistd::chdir(tempdir.path()); let dev = oci::LinuxDevice { path: "/fifo".to_string(), From 4dc3bc0020060335b8575c6e7ace28de6da9ab2d Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 24 Sep 2020 13:50:13 +0200 Subject: [PATCH 025/124] rust-agent: Treat warnings as error Avoid the accumulation of warnings we had, as reported in #750. Fixes: #750 Signed-off-by: Christophe de Dinechin --- src/agent/Makefile | 4 ++-- src/trace-forwarder/Makefile | 2 +- tools/agent-ctl/Makefile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent/Makefile b/src/agent/Makefile index b98b9510ef..8b97439e8f 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -106,10 +106,10 @@ default: $(TARGET) show-header $(TARGET): $(GENERATED_CODE) $(TARGET_PATH) $(TARGET_PATH): $(SOURCES) | show-summary - @cargo build --target $(TRIPLE) --$(BUILD_TYPE) + @RUSTFLAGS="--deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) optimize: $(SOURCES) | show-summary show-header - @RUSTFLAGS='-C link-arg=-s' cargo build --target $(TRIPLE) --$(BUILD_TYPE) + @RUSTFLAGS='-C link-arg=-s --deny-warnings' cargo build --target $(TRIPLE) --$(BUILD_TYPE) show-header: @printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)" diff --git a/src/trace-forwarder/Makefile b/src/trace-forwarder/Makefile index e02aef3975..ae73325922 100644 --- a/src/trace-forwarder/Makefile +++ b/src/trace-forwarder/Makefile @@ -6,7 +6,7 @@ default: build build: - cargo build -v + RUSTFLAGS="--deny warnings" cargo build -v clean: cargo clean diff --git a/tools/agent-ctl/Makefile b/tools/agent-ctl/Makefile index e02aef3975..ae73325922 100644 --- a/tools/agent-ctl/Makefile +++ b/tools/agent-ctl/Makefile @@ -6,7 +6,7 @@ default: build build: - cargo build -v + RUSTFLAGS="--deny warnings" cargo build -v clean: cargo clean From a116ce0b757f4e418ba6228126f5028050d3111e Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 8 Oct 2020 17:05:13 +0200 Subject: [PATCH 026/124] osbuilder: Create target directory for agent When building with AGENT_SOURCE_BIN pointing to an already built kata-agent binary, the target directory needs to be created in the rootfs tree. Fixes #873 Signed-off-by: Ralf Haferkamp --- tools/osbuilder/rootfs-builder/rootfs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 29682774c7..92c2e55829 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -559,6 +559,7 @@ EOT [ "$ARCH" == "aarch64" ] && export PATH=$OLD_PATH && rm -rf /usr/local/musl popd else + mkdir -p ${AGENT_DIR} cp ${AGENT_SOURCE_BIN} ${AGENT_DEST} OK "cp ${AGENT_SOURCE_BIN} ${AGENT_DEST}" fi From 6d80df98319869133da7478cc78942ea860622fa Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Thu, 8 Oct 2020 10:53:20 -0500 Subject: [PATCH 027/124] snap: specify python version In order to avoid `unmet dependencies` error in the CI, the python version must be specified in the yaml. fixes #877 Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 98f577f27a..37b2110f29 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -224,7 +224,7 @@ parts: after: [godeps, runtime] build-packages: - gcc - - python + - python3 - zlib1g-dev - libcap-ng-dev - libglib2.0-dev From 4f1d23b65197c3fda622a78d82125a25511b72bd Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Thu, 8 Oct 2020 10:59:10 -0500 Subject: [PATCH 028/124] virtiofs: Disable DAX virtiofs DAX support is not stable today, there are a few corner cases to make it default. Fixes: #862 Fixes: #875 Signed-off-by: Jose Carlos Venegas Munoz --- src/runtime/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 8cf73abeb9..2a6730d4ef 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -173,7 +173,8 @@ DEFSHAREDFS := virtio-9p DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd # Default DAX mapping cache size in MiB -DEFVIRTIOFSCACHESIZE := 1024 +#if value is 0, DAX is not enabled +DEFVIRTIOFSCACHESIZE := 0 DEFVIRTIOFSCACHE ?= auto # Format example: # [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] From eaff5de37a8a53622067222613349045903dbd8b Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Thu, 8 Oct 2020 11:39:42 -0500 Subject: [PATCH 029/124] versions: add plugins section plugins sections contains the details of plugins required for the components or testing. Add sriov-network-device-plugin url and version that are consumed by the VFIO test in the tests repository. fixes #879 Signed-off-by: Julio Montes --- versions.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/versions.yaml b/versions.yaml index 5be57d05e0..2ac20ba802 100644 --- a/versions.yaml +++ b/versions.yaml @@ -301,3 +301,15 @@ specs: https://github.com/opencontainers/runtime-spec/tags .*/v?(\d\S+)\.tar\.gz version: "v1.0.0-rc5" + +plugins: + description: | + Details of plugins required for the components or testing. + + sriov-network-device: + description: | + The SR-IOV network device plugin is Kubernetes device plugin for + discovering and advertising SR-IOV virtual functions (VFs) + available on a Kubernetes host. + url: "https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin" + version: "b7f6d3e0679796e907ecca88cfab0e32e326850d" From 2a6c9eec744f8b2dfe5551980b81da592bc20cfa Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 9 Oct 2020 10:50:45 +0800 Subject: [PATCH 030/124] agent-ctl: include cargo lock updates Simply running `make` would generate some cargo lock updates for agent-ctl. Let's include them so that we have fixed dependencies. Fixes: #883 Signed-off-by: Peng Tao --- tools/agent-ctl/Cargo.lock | 52 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/tools/agent-ctl/Cargo.lock b/tools/agent-ctl/Cargo.lock index ea06728c34..897425ab5c 100644 --- a/tools/agent-ctl/Cargo.lock +++ b/tools/agent-ctl/Cargo.lock @@ -111,6 +111,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cgroups" +version = "0.1.1-alpha.0" +source = "git+https://github.com/kata-containers/cgroups-rs?branch=stable-0.1.1#8717524f2c95aacd30768b6f0f7d7f2fddef5cac" +dependencies = [ + "libc", + "log", + "nix 0.18.0", + "regex", +] + [[package]] name = "chrono" version = "0.4.11" @@ -289,9 +300,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" [[package]] name = "log" @@ -351,6 +362,18 @@ dependencies = [ "void", ] +[[package]] +name = "nix" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83450fe6a6142ddd95fb064b746083fc4ef1705fe81f64a64e1d4b39f54a1055" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", +] + [[package]] name = "num-integer" version = "0.1.43" @@ -543,6 +566,15 @@ version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rust-argon2" version = "0.7.0" @@ -561,6 +593,7 @@ version = "0.1.0" dependencies = [ "anyhow", "caps", + "cgroups", "dirs", "lazy_static", "libc", @@ -578,6 +611,7 @@ dependencies = [ "serde_json", "slog", "slog-scope", + "tempfile", ] [[package]] @@ -702,6 +736,20 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "textwrap" version = "0.11.0" From cb2255f199d61a8a2706865bcdbbf787074024c4 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 9 Oct 2020 15:09:16 +0800 Subject: [PATCH 031/124] agent: set init process non-dumpable On old kernels (like v4.9), kernel applies CLOECEC in wrong order w.r.t. dumpable task flags. As a result, we might leak guest file descriptor to containers. This is a former runc CVE-2016-9962 and still applies to kata agent. Although Kata container is still valid at protecting the host, we should not leak extra resources to user containers. This sets the init processes that join and setup the container's namespaces as non-dumpable before they setns to the container's pid (or any other ) namespace. This settings is automatically reset to the default after the Exec in the container so that it does not change functionality for the applications that are running inside, just our init processes. This prevents parent processes, the pid 1 of the container, to ptrace the init process before it drops caps and other sets LSMs. The order during the exec syscall is that the process is set back to dumpable before O_CLOEXEC are processed. Refs: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=613cc2b6f272c1a8ad33aefa21cad77af23139f7 https://github.com/torvalds/linux/blob/v4.9/fs/exec.c#L1290-L1318 opencontainers/runc@50a19c6 https://nvd.nist.gov/vuln/detail/CVE-2016-9962 Fixes: #890 Signed-off-by: Peng Tao --- src/agent/rustjail/src/container.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index e30f4d420a..c6d14e63cd 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -456,6 +456,24 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { setrlimit(rl)?; } + // + // Make the process non-dumpable, to avoid various race conditions that + // could cause processes in namespaces we're joining to access host + // resources (or potentially execute code). + // + // However, if the number of namespaces we are joining is 0, we are not + // going to be switching to a different security context. Thus setting + // ourselves to be non-dumpable only breaks things (like rootless + // containers), which is the recommendation from the kernel folks. + // + // Ref: https://github.com/opencontainers/runc/commit/50a19c6ff828c58e5dab13830bd3dacde268afe5 + // + if !nses.is_empty() { + if let Err(e) = prctl::set_dumpable(false) { + return Err(anyhow!(e).context("set process non-dumpable failed")); + }; + } + if userns { log_child!(cfd_log, "enter new user namespace"); sched::unshare(CloneFlags::CLONE_NEWUSER)?; From 999f67d5732a8a04f9a6fdc0471f46241be317d9 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Fri, 9 Oct 2020 12:04:48 +0800 Subject: [PATCH 032/124] agent: do not follow link when mounting container proc and sysfs Attackers might use it to explore other containers in the same pod. While it is still safe to allow it, we can just close the race window like runc does. Fixes: #885 Signed-off-by: Peng Tao --- src/agent/rustjail/src/mount.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index 4256a9832d..b45ca97cea 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -205,6 +205,21 @@ pub fn init_rootfs( check_proc_mount(m)?; } + // If the destination already exists and is not a directory, we bail + // out This is to avoid mounting through a symlink or similar -- which + // has been a "fun" attack scenario in the past. + if m.r#type == "proc" || m.r#type == "sysfs" { + if let Ok(meta) = fs::symlink_metadata(&m.destination) { + if !meta.is_dir() { + return Err(anyhow!( + "Mount point {} must be ordinary directory: got {:?}", + m.destination, + meta.file_type() + )); + } + } + } + mount_from(cfd_log, &m, &rootfs, flags, &data, "")?; // bind mount won't change mount options, we need remount to make mount options // effective. From 1789527d611ca06db256b0e613b2a05e44244eef Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 9 Oct 2020 08:30:29 -0500 Subject: [PATCH 033/124] ci: snap: add event filtering Run the snap CI on every PR is not needed. Don't run the snap CI on PRs that don't change the source code (*.go/*.rs), a configuration file or Makefile. fixes #896 Signed-off-by: Julio Montes --- .github/workflows/snap.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml index 40abbde98a..8c7f946b95 100644 --- a/.github/workflows/snap.yaml +++ b/.github/workflows/snap.yaml @@ -1,5 +1,15 @@ name: snap CI -on: ["pull_request"] +on: + pull_request: + paths: + - "**/Makefile" + - "**/*.go" + - "**/*.mk" + - "**/*.rs" + - "**/*.sh" + - "**/*.toml" + - "**/*.yaml" + - "**/*.yml" jobs: test: runs-on: ubuntu-20.04 From f60f43af6b4974e6c31b904d61c8bf2019589715 Mon Sep 17 00:00:00 2001 From: Ychau Wang Date: Fri, 18 Sep 2020 13:51:02 +0800 Subject: [PATCH 034/124] runtime: Clear the VCMock 1.x API Methods from 2.0 Clear the 1.x branch api methods in the 2.0. Keep the same methods to the VC interface, like the VCImpl struct. Fixes: #751 Signed-off-by: Ychau Wang --- src/runtime/containerd-shim-v2/create_test.go | 6 +- .../containerd-shim-v2/metrics_test.go | 14 +- src/runtime/containerd-shim-v2/pause_test.go | 32 +- src/runtime/containerd-shim-v2/start_test.go | 28 +- src/runtime/virtcontainers/pkg/vcmock/mock.go | 241 ------- .../virtcontainers/pkg/vcmock/mock_test.go | 626 +----------------- .../virtcontainers/pkg/vcmock/types.go | 29 - 7 files changed, 57 insertions(+), 919 deletions(-) diff --git a/src/runtime/containerd-shim-v2/create_test.go b/src/runtime/containerd-shim-v2/create_test.go index 7a67b78d12..d2e0b58244 100644 --- a/src/runtime/containerd-shim-v2/create_test.go +++ b/src/runtime/containerd-shim-v2/create_test.go @@ -317,12 +317,12 @@ func TestCreateContainerConfigFail(t *testing.T) { MockID: testSandboxID, } - testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) { - return sandbox, &vcmock.Container{}, nil + sandbox.CreateContainerFunc = func(conf vc.ContainerConfig) (vc.VCContainer, error) { + return &vcmock.Container{}, nil } defer func() { - testingImpl.CreateContainerFunc = nil + sandbox.CreateContainerFunc = nil }() tmpdir, err := ioutil.TempDir("", "") diff --git a/src/runtime/containerd-shim-v2/metrics_test.go b/src/runtime/containerd-shim-v2/metrics_test.go index a65aec982f..a2ff047d9a 100644 --- a/src/runtime/containerd-shim-v2/metrics_test.go +++ b/src/runtime/containerd-shim-v2/metrics_test.go @@ -7,12 +7,11 @@ package containerdshim import ( - "context" "testing" "github.com/containerd/cgroups" - "github.com/containerd/containerd/namespaces" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock" "github.com/stretchr/testify/assert" ) @@ -37,18 +36,21 @@ func TestStatNetworkMetric(t *testing.T) { }, } - testingImpl.StatsContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) { + sandbox := &vcmock.Sandbox{ + MockID: testSandboxID, + } + + sandbox.StatsContainerFunc = func(contID string) (vc.ContainerStats, error) { return vc.ContainerStats{ NetworkStats: mockNetwork, }, nil } defer func() { - testingImpl.StatsContainerFunc = nil + sandbox.StatsContainerFunc = nil }() - ctx := namespaces.WithNamespace(context.Background(), "UnitTest") - resp, err := testingImpl.StatsContainer(ctx, testSandboxID, testContainerID) + resp, err := sandbox.StatsContainer(testContainerID) assert.NoError(err) metrics := statsToMetrics(&resp) diff --git a/src/runtime/containerd-shim-v2/pause_test.go b/src/runtime/containerd-shim-v2/pause_test.go index 56d6994d5f..6cf7227ef1 100644 --- a/src/runtime/containerd-shim-v2/pause_test.go +++ b/src/runtime/containerd-shim-v2/pause_test.go @@ -28,14 +28,14 @@ func TestPauseContainerSuccess(t *testing.T) { MockID: testSandboxID, } - testingImpl.PauseContainerFunc = func(ctx context.Context, sandboxID, containerID string) error { + sandbox.PauseContainerFunc = func(contID string) error { return nil } defer func() { - testingImpl.PauseContainerFunc = nil + sandbox.PauseContainerFunc = nil }() - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: testContainerID, Annotations: make(map[string]string), @@ -45,7 +45,7 @@ func TestPauseContainerSuccess(t *testing.T) { }, nil } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ @@ -76,14 +76,14 @@ func TestPauseContainerFail(t *testing.T) { MockID: testSandboxID, } - testingImpl.PauseContainerFunc = func(ctx context.Context, sandboxID, containerID string) error { + sandbox.PauseContainerFunc = func(contID string) error { return nil } defer func() { - testingImpl.PauseContainerFunc = nil + sandbox.PauseContainerFunc = nil }() - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: testContainerID, Annotations: make(map[string]string), @@ -93,7 +93,7 @@ func TestPauseContainerFail(t *testing.T) { }, nil } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ @@ -119,14 +119,14 @@ func TestResumeContainerSuccess(t *testing.T) { MockID: testSandboxID, } - testingImpl.ResumeContainerFunc = func(ctx context.Context, sandboxID, containerID string) error { + sandbox.ResumeContainerFunc = func(contID string) error { return nil } defer func() { - testingImpl.ResumeContainerFunc = nil + sandbox.ResumeContainerFunc = nil }() - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: testContainerID, Annotations: make(map[string]string), @@ -137,7 +137,7 @@ func TestResumeContainerSuccess(t *testing.T) { } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ @@ -168,13 +168,13 @@ func TestResumeContainerFail(t *testing.T) { MockID: testSandboxID, } - testingImpl.ResumeContainerFunc = func(ctx context.Context, sandboxID, containerID string) error { + sandbox.ResumeContainerFunc = func(contID string) error { return nil } defer func() { - testingImpl.ResumeContainerFunc = nil + sandbox.ResumeContainerFunc = nil }() - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: testContainerID, Annotations: make(map[string]string), @@ -184,7 +184,7 @@ func TestResumeContainerFail(t *testing.T) { }, nil } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ diff --git a/src/runtime/containerd-shim-v2/start_test.go b/src/runtime/containerd-shim-v2/start_test.go index 22a024e5d4..c2ed893be6 100644 --- a/src/runtime/containerd-shim-v2/start_test.go +++ b/src/runtime/containerd-shim-v2/start_test.go @@ -28,7 +28,7 @@ func TestStartStartSandboxSuccess(t *testing.T) { MockID: testSandboxID, } - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: sandbox.ID(), Annotations: map[string]string{ @@ -38,7 +38,7 @@ func TestStartStartSandboxSuccess(t *testing.T) { } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ @@ -58,12 +58,12 @@ func TestStartStartSandboxSuccess(t *testing.T) { ID: testSandboxID, } - testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return sandbox, nil + sandbox.StartFunc = func() error { + return nil } defer func() { - testingImpl.StartSandboxFunc = nil + sandbox.StartFunc = nil }() ctx := namespaces.WithNamespace(context.Background(), "UnitTest") @@ -79,7 +79,7 @@ func TestStartMissingAnnotation(t *testing.T) { MockID: testSandboxID, } - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: sandbox.ID(), Annotations: map[string]string{}, @@ -87,7 +87,7 @@ func TestStartMissingAnnotation(t *testing.T) { } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() s := &service{ @@ -107,12 +107,12 @@ func TestStartMissingAnnotation(t *testing.T) { ID: testSandboxID, } - testingImpl.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return sandbox, nil + sandbox.StartFunc = func() error { + return nil } defer func() { - testingImpl.StartSandboxFunc = nil + sandbox.StartFunc = nil }() _, err = s.Start(s.ctx, reqStart) @@ -135,7 +135,7 @@ func TestStartStartContainerSucess(t *testing.T) { }, } - testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { + sandbox.StatusContainerFunc = func(contID string) (vc.ContainerStatus, error) { return vc.ContainerStatus{ ID: testContainerID, Annotations: map[string]string{ @@ -145,15 +145,15 @@ func TestStartStartContainerSucess(t *testing.T) { } defer func() { - testingImpl.StatusContainerFunc = nil + sandbox.StatusContainerFunc = nil }() - testingImpl.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { + sandbox.StartContainerFunc = func(contID string) (vc.VCContainer, error) { return sandbox.MockContainers[0], nil } defer func() { - testingImpl.StartContainerFunc = nil + sandbox.StartContainerFunc = nil }() s := &service{ diff --git a/src/runtime/virtcontainers/pkg/vcmock/mock.go b/src/runtime/virtcontainers/pkg/vcmock/mock.go index b75292e8eb..5a13c76c45 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/mock.go +++ b/src/runtime/virtcontainers/pkg/vcmock/mock.go @@ -18,14 +18,7 @@ package vcmock import ( "context" "fmt" - "syscall" - vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -56,240 +49,6 @@ func (m *VCMock) CreateSandbox(ctx context.Context, sandboxConfig vc.SandboxConf return nil, fmt.Errorf("%s: %s (%+v): sandboxConfig: %v", mockErrorPrefix, getSelf(), m, sandboxConfig) } -// DeleteSandbox implements the VC function of the same name. -func (m *VCMock) DeleteSandbox(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - if m.DeleteSandboxFunc != nil { - return m.DeleteSandboxFunc(ctx, sandboxID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// FetchSandbox implements the VC function of the same name. -func (m *VCMock) FetchSandbox(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - if m.FetchSandboxFunc != nil { - return m.FetchSandboxFunc(ctx, sandboxID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// StartSandbox implements the VC function of the same name. -func (m *VCMock) StartSandbox(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - if m.StartSandboxFunc != nil { - return m.StartSandboxFunc(ctx, sandboxID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// StopSandbox implements the VC function of the same name. -func (m *VCMock) StopSandbox(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) { - if m.StopSandboxFunc != nil { - return m.StopSandboxFunc(ctx, sandboxID, force) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// RunSandbox implements the VC function of the same name. -func (m *VCMock) RunSandbox(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { - if m.RunSandboxFunc != nil { - return m.RunSandboxFunc(ctx, sandboxConfig) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxConfig: %v", mockErrorPrefix, getSelf(), m, sandboxConfig) -} - -// ListSandbox implements the VC function of the same name. -func (m *VCMock) ListSandbox(ctx context.Context) ([]vc.SandboxStatus, error) { - if m.ListSandboxFunc != nil { - return m.ListSandboxFunc(ctx) - } - - return nil, fmt.Errorf("%s: %s", mockErrorPrefix, getSelf()) -} - -// StatusSandbox implements the VC function of the same name. -func (m *VCMock) StatusSandbox(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { - if m.StatusSandboxFunc != nil { - return m.StatusSandboxFunc(ctx, sandboxID) - } - - return vc.SandboxStatus{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// CreateContainer implements the VC function of the same name. -func (m *VCMock) CreateContainer(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) { - if m.CreateContainerFunc != nil { - return m.CreateContainerFunc(ctx, sandboxID, containerConfig) - } - - return nil, nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerConfig: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerConfig) -} - -// DeleteContainer implements the VC function of the same name. -func (m *VCMock) DeleteContainer(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - if m.DeleteContainerFunc != nil { - return m.DeleteContainerFunc(ctx, sandboxID, containerID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// StartContainer implements the VC function of the same name. -func (m *VCMock) StartContainer(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - if m.StartContainerFunc != nil { - return m.StartContainerFunc(ctx, sandboxID, containerID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// StopContainer implements the VC function of the same name. -func (m *VCMock) StopContainer(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - if m.StopContainerFunc != nil { - return m.StopContainerFunc(ctx, sandboxID, containerID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// EnterContainer implements the VC function of the same name. -func (m *VCMock) EnterContainer(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) { - if m.EnterContainerFunc != nil { - return m.EnterContainerFunc(ctx, sandboxID, containerID, cmd) - } - - return nil, nil, nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v, cmd: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID, cmd) -} - -// StatusContainer implements the VC function of the same name. -func (m *VCMock) StatusContainer(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { - if m.StatusContainerFunc != nil { - return m.StatusContainerFunc(ctx, sandboxID, containerID) - } - - return vc.ContainerStatus{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// StatsContainer implements the VC function of the same name. -func (m *VCMock) StatsContainer(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) { - if m.StatsContainerFunc != nil { - return m.StatsContainerFunc(ctx, sandboxID, containerID) - } - - return vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// StatsSandbox implements the VC function of the same name. -func (m *VCMock) StatsSandbox(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) { - if m.StatsContainerFunc != nil { - return m.StatsSandboxFunc(ctx, sandboxID) - } - - return vc.SandboxStats{}, []vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// KillContainer implements the VC function of the same name. -func (m *VCMock) KillContainer(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { - if m.KillContainerFunc != nil { - return m.KillContainerFunc(ctx, sandboxID, containerID, signal, all) - } - - return fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v, signal: %v, all: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID, signal, all) -} - -// ProcessListContainer implements the VC function of the same name. -func (m *VCMock) ProcessListContainer(ctx context.Context, sandboxID, containerID string, options vc.ProcessListOptions) (vc.ProcessList, error) { - if m.ProcessListContainerFunc != nil { - return m.ProcessListContainerFunc(ctx, sandboxID, containerID, options) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// UpdateContainer implements the VC function of the same name. -func (m *VCMock) UpdateContainer(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error { - if m.UpdateContainerFunc != nil { - return m.UpdateContainerFunc(ctx, sandboxID, containerID, resources) - } - - return fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// PauseContainer implements the VC function of the same name. -func (m *VCMock) PauseContainer(ctx context.Context, sandboxID, containerID string) error { - if m.PauseContainerFunc != nil { - return m.PauseContainerFunc(ctx, sandboxID, containerID) - } - - return fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// ResumeContainer implements the VC function of the same name. -func (m *VCMock) ResumeContainer(ctx context.Context, sandboxID, containerID string) error { - if m.ResumeContainerFunc != nil { - return m.ResumeContainerFunc(ctx, sandboxID, containerID) - } - - return fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) -} - -// AddDevice implements the VC function of the same name. -func (m *VCMock) AddDevice(ctx context.Context, sandboxID string, info config.DeviceInfo) (api.Device, error) { - if m.AddDeviceFunc != nil { - return m.AddDeviceFunc(ctx, sandboxID, info) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// AddInterface implements the VC function of the same name. -func (m *VCMock) AddInterface(ctx context.Context, sandboxID string, inf *pbTypes.Interface) (*pbTypes.Interface, error) { - if m.AddInterfaceFunc != nil { - return m.AddInterfaceFunc(ctx, sandboxID, inf) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// RemoveInterface implements the VC function of the same name. -func (m *VCMock) RemoveInterface(ctx context.Context, sandboxID string, inf *pbTypes.Interface) (*pbTypes.Interface, error) { - if m.RemoveInterfaceFunc != nil { - return m.RemoveInterfaceFunc(ctx, sandboxID, inf) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// ListInterfaces implements the VC function of the same name. -func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*pbTypes.Interface, error) { - if m.ListInterfacesFunc != nil { - return m.ListInterfacesFunc(ctx, sandboxID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// UpdateRoutes implements the VC function of the same name. -func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*pbTypes.Route) ([]*pbTypes.Route, error) { - if m.UpdateRoutesFunc != nil { - return m.UpdateRoutesFunc(ctx, sandboxID, routes) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - -// ListRoutes implements the VC function of the same name. -func (m *VCMock) ListRoutes(ctx context.Context, sandboxID string) ([]*pbTypes.Route, error) { - if m.ListRoutesFunc != nil { - return m.ListRoutesFunc(ctx, sandboxID) - } - - return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) -} - func (m *VCMock) CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error { if m.CleanupContainerFunc != nil { return m.CleanupContainerFunc(ctx, sandboxID, containerID, true) diff --git a/src/runtime/virtcontainers/pkg/vcmock/mock_test.go b/src/runtime/virtcontainers/pkg/vcmock/mock_test.go index 0a0292bfc0..9043b168da 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/mock_test.go +++ b/src/runtime/virtcontainers/pkg/vcmock/mock_test.go @@ -8,13 +8,10 @@ package vcmock import ( "context" "reflect" - "syscall" "testing" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory" - pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -143,514 +140,6 @@ func TestVCMockCreateSandbox(t *testing.T) { assert.True(IsMockError(err)) } -func TestVCMockDeleteSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.DeleteSandboxFunc) - - ctx := context.Background() - _, err := m.DeleteSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return &Sandbox{}, nil - } - - sandbox, err := m.DeleteSandbox(ctx, testSandboxID) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - - // reset - m.DeleteSandboxFunc = nil - - _, err = m.DeleteSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockListSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.ListSandboxFunc) - - ctx := context.Background() - _, err := m.ListSandbox(ctx) - assert.Error(err) - assert.True(IsMockError(err)) - - m.ListSandboxFunc = func(ctx context.Context) ([]vc.SandboxStatus, error) { - return []vc.SandboxStatus{}, nil - } - - sandboxes, err := m.ListSandbox(ctx) - assert.NoError(err) - assert.Equal(sandboxes, []vc.SandboxStatus{}) - - // reset - m.ListSandboxFunc = nil - - _, err = m.ListSandbox(ctx) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockRunSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.RunSandboxFunc) - - ctx := context.Background() - _, err := m.RunSandbox(ctx, vc.SandboxConfig{}) - assert.Error(err) - assert.True(IsMockError(err)) - - m.RunSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { - return &Sandbox{}, nil - } - - sandbox, err := m.RunSandbox(ctx, vc.SandboxConfig{}) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - - // reset - m.RunSandboxFunc = nil - - _, err = m.RunSandbox(ctx, vc.SandboxConfig{}) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStartSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StartSandboxFunc) - - ctx := context.Background() - _, err := m.StartSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StartSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { - return &Sandbox{}, nil - } - - sandbox, err := m.StartSandbox(ctx, testSandboxID) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - - // reset - m.StartSandboxFunc = nil - - _, err = m.StartSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStatusSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StatusSandboxFunc) - - ctx := context.Background() - _, err := m.StatusSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { - return vc.SandboxStatus{}, nil - } - - sandbox, err := m.StatusSandbox(ctx, testSandboxID) - assert.NoError(err) - assert.Equal(sandbox, vc.SandboxStatus{}) - - // reset - m.StatusSandboxFunc = nil - - _, err = m.StatusSandbox(ctx, testSandboxID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStopSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StopSandboxFunc) - - ctx := context.Background() - _, err := m.StopSandbox(ctx, testSandboxID, false) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) { - return &Sandbox{}, nil - } - - sandbox, err := m.StopSandbox(ctx, testSandboxID, false) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - - // reset - m.StopSandboxFunc = nil - - _, err = m.StopSandbox(ctx, testSandboxID, false) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockCreateContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.CreateContainerFunc) - - ctx := context.Background() - config := vc.ContainerConfig{} - _, _, err := m.CreateContainer(ctx, testSandboxID, config) - assert.Error(err) - assert.True(IsMockError(err)) - - m.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) { - return &Sandbox{}, &Container{}, nil - } - - sandbox, container, err := m.CreateContainer(ctx, testSandboxID, config) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - assert.Equal(container, &Container{}) - - // reset - m.CreateContainerFunc = nil - - _, _, err = m.CreateContainer(ctx, testSandboxID, config) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockDeleteContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.DeleteContainerFunc) - - ctx := context.Background() - _, err := m.DeleteContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.DeleteContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - return &Container{}, nil - } - - container, err := m.DeleteContainer(ctx, testSandboxID, testContainerID) - assert.NoError(err) - assert.Equal(container, &Container{}) - - // reset - m.DeleteContainerFunc = nil - - _, err = m.DeleteContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockEnterContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.EnterContainerFunc) - - ctx := context.Background() - cmd := types.Cmd{} - _, _, _, err := m.EnterContainer(ctx, testSandboxID, testContainerID, cmd) - assert.Error(err) - assert.True(IsMockError(err)) - - m.EnterContainerFunc = func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) { - return &Sandbox{}, &Container{}, &vc.Process{}, nil - } - - sandbox, container, process, err := m.EnterContainer(ctx, testSandboxID, testContainerID, cmd) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - assert.Equal(container, &Container{}) - assert.Equal(process, &vc.Process{}) - - // reset - m.EnterContainerFunc = nil - - _, _, _, err = m.EnterContainer(ctx, testSandboxID, testContainerID, cmd) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockKillContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.KillContainerFunc) - - ctx := context.Background() - sig := syscall.SIGTERM - - for _, all := range []bool{true, false} { - err := m.KillContainer(ctx, testSandboxID, testContainerID, sig, all) - assert.Error(err) - assert.True(IsMockError(err)) - } - - m.KillContainerFunc = func(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { - return nil - } - - for _, all := range []bool{true, false} { - err := m.KillContainer(ctx, testSandboxID, testContainerID, sig, all) - assert.NoError(err) - } - - // reset - m.KillContainerFunc = nil - - for _, all := range []bool{true, false} { - err := m.KillContainer(ctx, testSandboxID, testContainerID, sig, all) - assert.Error(err) - assert.True(IsMockError(err)) - } -} - -func TestVCMockStartContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StartContainerFunc) - - ctx := context.Background() - _, err := m.StartContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StartContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - return &Container{}, nil - } - - container, err := m.StartContainer(ctx, testSandboxID, testContainerID) - assert.NoError(err) - assert.Equal(container, &Container{}) - - // reset - m.StartContainerFunc = nil - - _, err = m.StartContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStatusContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StatusContainerFunc) - - ctx := context.Background() - _, err := m.StatusContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { - return vc.ContainerStatus{}, nil - } - - status, err := m.StatusContainer(ctx, testSandboxID, testContainerID) - assert.NoError(err) - assert.Equal(status, vc.ContainerStatus{}) - - // reset - m.StatusContainerFunc = nil - - _, err = m.StatusContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStatsContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StatsContainerFunc) - - ctx := context.Background() - _, err := m.StatsContainer(ctx, testSandboxID, testContainerID) - - assert.Error(err) - assert.True(IsMockError(err)) - - m.StatsContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) { - return vc.ContainerStats{}, nil - } - - stats, err := m.StatsContainer(ctx, testSandboxID, testContainerID) - assert.NoError(err) - assert.Equal(stats, vc.ContainerStats{}) - - // reset - m.StatsContainerFunc = nil - - _, err = m.StatsContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockStopContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.StopContainerFunc) - - ctx := context.Background() - _, err := m.StopContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.StopContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) { - return &Container{}, nil - } - - container, err := m.StopContainer(ctx, testSandboxID, testContainerID) - assert.NoError(err) - assert.Equal(container, &Container{}) - - // reset - m.StopContainerFunc = nil - - _, err = m.StopContainer(ctx, testSandboxID, testContainerID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockProcessListContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - assert.Nil(m.ProcessListContainerFunc) - - options := vc.ProcessListOptions{ - Format: "json", - Args: []string{"-ef"}, - } - - ctx := context.Background() - _, err := m.ProcessListContainer(ctx, testSandboxID, testContainerID, options) - assert.Error(err) - assert.True(IsMockError(err)) - - processList := vc.ProcessList("hi") - - m.ProcessListContainerFunc = func(ctx context.Context, sandboxID, containerID string, options vc.ProcessListOptions) (vc.ProcessList, error) { - return processList, nil - } - - pList, err := m.ProcessListContainer(ctx, testSandboxID, testContainerID, options) - assert.NoError(err) - assert.Equal(pList, processList) - - // reset - m.ProcessListContainerFunc = nil - - _, err = m.ProcessListContainer(ctx, testSandboxID, testContainerID, options) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockFetchSandbox(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.FetchSandboxFunc) - - ctx := context.Background() - _, err := m.FetchSandbox(ctx, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) { - return &Sandbox{}, nil - } - - sandbox, err := m.FetchSandbox(ctx, config.ID) - assert.NoError(err) - assert.Equal(sandbox, &Sandbox{}) - - // reset - m.FetchSandboxFunc = nil - - _, err = m.FetchSandbox(ctx, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - -} - -func TestVCMockPauseContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.PauseContainerFunc) - - ctx := context.Background() - err := m.PauseContainer(ctx, config.ID, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.PauseContainerFunc = func(ctx context.Context, sid, cid string) error { - return nil - } - - err = m.PauseContainer(ctx, config.ID, config.ID) - assert.NoError(err) - - // reset - m.PauseContainerFunc = nil - - err = m.PauseContainer(ctx, config.ID, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockResumeContainer(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.ResumeContainerFunc) - - ctx := context.Background() - err := m.ResumeContainer(ctx, config.ID, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.ResumeContainerFunc = func(ctx context.Context, sid, cid string) error { - return nil - } - - err = m.ResumeContainer(ctx, config.ID, config.ID) - assert.NoError(err) - - // reset - m.ResumeContainerFunc = nil - - err = m.ResumeContainer(ctx, config.ID, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) -} - func TestVCMockSetVMFactory(t *testing.T) { assert := assert.New(t) @@ -682,137 +171,54 @@ func TestVCMockSetVMFactory(t *testing.T) { assert.Equal(factoryTriggered, 1) } -func TestVCMockAddInterface(t *testing.T) { +func TestVCMockCleanupContainer(t *testing.T) { assert := assert.New(t) m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.AddInterfaceFunc) + assert.Nil(m.CleanupContainerFunc) ctx := context.Background() - _, err := m.AddInterface(ctx, config.ID, nil) + err := m.CleanupContainer(ctx, testSandboxID, testContainerID, false) assert.Error(err) assert.True(IsMockError(err)) - m.AddInterfaceFunc = func(ctx context.Context, sid string, inf *pbTypes.Interface) (*pbTypes.Interface, error) { - return nil, nil + m.CleanupContainerFunc = func(ctx context.Context, sandboxID, containerID string, force bool) error { + return nil } - _, err = m.AddInterface(ctx, config.ID, nil) + err = m.CleanupContainer(ctx, testSandboxID, testContainerID, false) assert.NoError(err) // reset - m.AddInterfaceFunc = nil + m.CleanupContainerFunc = nil - _, err = m.AddInterface(ctx, config.ID, nil) + err = m.CleanupContainer(ctx, testSandboxID, testContainerID, false) assert.Error(err) assert.True(IsMockError(err)) } -func TestVCMockRemoveInterface(t *testing.T) { +func TestVCMockForceCleanupContainer(t *testing.T) { assert := assert.New(t) m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.RemoveInterfaceFunc) + assert.Nil(m.CleanupContainerFunc) ctx := context.Background() - _, err := m.RemoveInterface(ctx, config.ID, nil) + err := m.CleanupContainer(ctx, testSandboxID, testContainerID, true) assert.Error(err) assert.True(IsMockError(err)) - m.RemoveInterfaceFunc = func(ctx context.Context, sid string, inf *pbTypes.Interface) (*pbTypes.Interface, error) { - return nil, nil + m.CleanupContainerFunc = func(ctx context.Context, sandboxID, containerID string, force bool) error { + return nil } - _, err = m.RemoveInterface(ctx, config.ID, nil) + err = m.CleanupContainer(ctx, testSandboxID, testContainerID, true) assert.NoError(err) // reset - m.RemoveInterfaceFunc = nil + m.CleanupContainerFunc = nil - _, err = m.RemoveInterface(ctx, config.ID, nil) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockListInterfaces(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.ListInterfacesFunc) - - ctx := context.Background() - _, err := m.ListInterfaces(ctx, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.ListInterfacesFunc = func(ctx context.Context, sid string) ([]*pbTypes.Interface, error) { - return nil, nil - } - - _, err = m.ListInterfaces(ctx, config.ID) - assert.NoError(err) - - // reset - m.ListInterfacesFunc = nil - - _, err = m.ListInterfaces(ctx, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockUpdateRoutes(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.UpdateRoutesFunc) - - ctx := context.Background() - _, err := m.UpdateRoutes(ctx, config.ID, nil) - assert.Error(err) - assert.True(IsMockError(err)) - - m.UpdateRoutesFunc = func(ctx context.Context, sid string, routes []*pbTypes.Route) ([]*pbTypes.Route, error) { - return nil, nil - } - - _, err = m.UpdateRoutes(ctx, config.ID, nil) - assert.NoError(err) - - // reset - m.UpdateRoutesFunc = nil - - _, err = m.UpdateRoutes(ctx, config.ID, nil) - assert.Error(err) - assert.True(IsMockError(err)) -} - -func TestVCMockListRoutes(t *testing.T) { - assert := assert.New(t) - - m := &VCMock{} - config := &vc.SandboxConfig{} - assert.Nil(m.ListRoutesFunc) - - ctx := context.Background() - _, err := m.ListRoutes(ctx, config.ID) - assert.Error(err) - assert.True(IsMockError(err)) - - m.ListRoutesFunc = func(ctx context.Context, sid string) ([]*pbTypes.Route, error) { - return nil, nil - } - - _, err = m.ListRoutes(ctx, config.ID) - assert.NoError(err) - - // reset - m.ListRoutesFunc = nil - - _, err = m.ListRoutes(ctx, config.ID) + err = m.CleanupContainer(ctx, testSandboxID, testContainerID, true) assert.Error(err) assert.True(IsMockError(err)) } diff --git a/src/runtime/virtcontainers/pkg/vcmock/types.go b/src/runtime/virtcontainers/pkg/vcmock/types.go index 211d97c8e2..9caa53232a 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/types.go +++ b/src/runtime/virtcontainers/pkg/vcmock/types.go @@ -88,34 +88,5 @@ type VCMock struct { SetFactoryFunc func(ctx context.Context, factory vc.Factory) CreateSandboxFunc func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) - DeleteSandboxFunc func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) - ListSandboxFunc func(ctx context.Context) ([]vc.SandboxStatus, error) - FetchSandboxFunc func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) - RunSandboxFunc func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) - StartSandboxFunc func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) - StatusSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) - StatsContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) - StatsSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) - StopSandboxFunc func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) - - CreateContainerFunc func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) - DeleteContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) - EnterContainerFunc func(ctx context.Context, sandboxID, containerID string, cmd types.Cmd) (vc.VCSandbox, vc.VCContainer, *vc.Process, error) - KillContainerFunc func(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error - StartContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) - StatusContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) - StopContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.VCContainer, error) - ProcessListContainerFunc func(ctx context.Context, sandboxID, containerID string, options vc.ProcessListOptions) (vc.ProcessList, error) - UpdateContainerFunc func(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error - PauseContainerFunc func(ctx context.Context, sandboxID, containerID string) error - ResumeContainerFunc func(ctx context.Context, sandboxID, containerID string) error - - AddDeviceFunc func(ctx context.Context, sandboxID string, info config.DeviceInfo) (api.Device, error) - - AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *pbTypes.Interface) (*pbTypes.Interface, error) - RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *pbTypes.Interface) (*pbTypes.Interface, error) - ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*pbTypes.Interface, error) - UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*pbTypes.Route) ([]*pbTypes.Route, error) - ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*pbTypes.Route, error) CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error } From 997f1f6cd0c115aec0bf9a97c736bd3d8e7da3f0 Mon Sep 17 00:00:00 2001 From: bin liu Date: Fri, 9 Oct 2020 14:22:57 +0800 Subject: [PATCH 035/124] docs: Add crictl example json files Add basic sample pod/container config files to show how to use `crictl` with Kata containers. Fixes: #881 Signed-off-by: bin liu --- docs/how-to/README.md | 1 + .../data/crictl/busybox/container_config.json | 18 +++ .../data/crictl/busybox/sandbox_config.json | 19 +++ .../redis/redis_client_container_config.json | 38 +++++ .../redis/redis_client_sandbox_config.json | 27 ++++ .../redis/redis_server_container_config.json | 35 ++++ .../redis/redis_server_sandbox_config.json | 27 ++++ docs/how-to/run-kata-with-crictl.md | 150 ++++++++++++++++++ 8 files changed, 315 insertions(+) create mode 100644 docs/how-to/data/crictl/busybox/container_config.json create mode 100644 docs/how-to/data/crictl/busybox/sandbox_config.json create mode 100644 docs/how-to/data/crictl/redis/redis_client_container_config.json create mode 100644 docs/how-to/data/crictl/redis/redis_client_sandbox_config.json create mode 100644 docs/how-to/data/crictl/redis/redis_server_container_config.json create mode 100644 docs/how-to/data/crictl/redis/redis_server_sandbox_config.json create mode 100644 docs/how-to/run-kata-with-crictl.md diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 2aaec04356..79f57966f4 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -6,6 +6,7 @@ * [Advanced Topics](#advanced-topics) ## Kubernetes Integration +- [Run Kata containers with `crictl`](run-kata-with-crictl.md) - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md) - [How to use Kata Containers and Containerd](containerd-kata.md) - [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md) diff --git a/docs/how-to/data/crictl/busybox/container_config.json b/docs/how-to/data/crictl/busybox/container_config.json new file mode 100644 index 0000000000..5772496514 --- /dev/null +++ b/docs/how-to/data/crictl/busybox/container_config.json @@ -0,0 +1,18 @@ +{ + "metadata": { + "name": "busybox-container" + }, + "image": { + "image": "docker.io/library/busybox:latest" + }, + "command": [ + "sleep", + "9999" + ], + "args": [], + "working_dir": "/", + "log_path": "", + "stdin": false, + "stdin_once": false, + "tty": false +} diff --git a/docs/how-to/data/crictl/busybox/sandbox_config.json b/docs/how-to/data/crictl/busybox/sandbox_config.json new file mode 100644 index 0000000000..963db8633c --- /dev/null +++ b/docs/how-to/data/crictl/busybox/sandbox_config.json @@ -0,0 +1,19 @@ +{ + "metadata": { + "name": "busybox-pod", + "uid": "busybox-pod" + }, + "hostname": "busybox_host", + "log_directory": "", + "dns_config": { + }, + "port_mappings": [], + "resources": { + }, + "labels": { + }, + "annotations": { + }, + "linux": { + } +} diff --git a/docs/how-to/data/crictl/redis/redis_client_container_config.json b/docs/how-to/data/crictl/redis/redis_client_container_config.json new file mode 100644 index 0000000000..95c42248ec --- /dev/null +++ b/docs/how-to/data/crictl/redis/redis_client_container_config.json @@ -0,0 +1,38 @@ +{ + "metadata": { + "name": "redis-client" + }, + "image": { + "image": "docker.io/library/redis:6.0.8-alpine" + }, + "command": [ + "tail", "-f", "/dev/null" + ], + "envs": [ + { + "key": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "key": "TERM", + "value": "xterm" + } + ], + "labels": { + "tier": "backend" + }, + "annotations": { + "pod": "redis-client-pod" + }, + "log_path": "", + "stdin": false, + "stdin_once": false, + "tty": false, + "linux": { + "resources": { + "memory_limit_in_bytes": 524288000 + }, + "security_context": { + } + } +} diff --git a/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json b/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json new file mode 100644 index 0000000000..6613d3c0d9 --- /dev/null +++ b/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "name": "redis-client-pod", + "uid": "test-redis-client-pod" + }, + "hostname": "redis-client", + "log_directory": "", + "dns_config": { + "searches": [ + "8.8.8.8" + ] + }, + "port_mappings": [], + "resources": { + "cpu": { + "limits": 1, + "requests": 1 + } + }, + "labels": { + "tier": "backend" + }, + "annotations": { + }, + "linux": { + } +} diff --git a/docs/how-to/data/crictl/redis/redis_server_container_config.json b/docs/how-to/data/crictl/redis/redis_server_container_config.json new file mode 100644 index 0000000000..faf1f444d0 --- /dev/null +++ b/docs/how-to/data/crictl/redis/redis_server_container_config.json @@ -0,0 +1,35 @@ +{ + "metadata": { + "name": "redis-server" + }, + "image": { + "image": "docker.io/library/redis:6.0.8-alpine" + }, + "envs": [ + { + "key": "PATH", + "value": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + }, + { + "key": "TERM", + "value": "xterm" + } + ], + "labels": { + "tier": "backend" + }, + "annotations": { + "pod": "redis-server-pod" + }, + "log_path": "", + "stdin": false, + "stdin_once": false, + "tty": false, + "linux": { + "resources": { + "memory_limit_in_bytes": 524288000 + }, + "security_context": { + } + } +} diff --git a/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json b/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json new file mode 100644 index 0000000000..29af2d133d --- /dev/null +++ b/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "name": "redis-server-pod", + "uid": "test-redis-server-pod" + }, + "hostname": "redis-server", + "log_directory": "", + "dns_config": { + "searches": [ + "8.8.8.8" + ] + }, + "port_mappings": [], + "resources": { + "cpu": { + "limits": 1, + "requests": 1 + } + }, + "labels": { + "tier": "backend" + }, + "annotations": { + }, + "linux": { + } +} diff --git a/docs/how-to/run-kata-with-crictl.md b/docs/how-to/run-kata-with-crictl.md new file mode 100644 index 0000000000..b7c1b19886 --- /dev/null +++ b/docs/how-to/run-kata-with-crictl.md @@ -0,0 +1,150 @@ +# Working with `crictl` + +* [What's `cri-tools`](#whats-cri-tools) +* [Use `crictl` run Pods in Kata containers](#use-crictl-run-pods-in-kata-containers) + * [Run `busybox` Pod](#run-busybox-pod) + * [Run pod sandbox with config file](#run-pod-sandbox-with-config-file) + * [Create container in the pod sandbox with config file](#create-container-in-the-pod-sandbox-with-config-file) + * [Start container](#start-container) + * [Run `redis` Pod](#run-redis-pod) + * [Create `redis-server` Pod](#create-redis-server-pod) + * [Create `redis-client` Pod](#create-redis-client-pod) + * [Check `redis` server is working](#check-redis-server-is-working) + +## What's `cri-tools` + +[`cri-tools`](https://github.com/kubernetes-sigs/cri-tools) provides debugging and validation tools for Kubelet Container Runtime Interface (CRI). + +`cri-tools` includes two tools: `crictl` and `critest`. `crictl` is the CLI for Kubelet CRI, in this document, we will show how to use `crictl` to run Pods in Kata containers. + +> **Note:** `cri-tools` is only used for debugging and validation purpose, and don't use it to run production workloads. + +> **Note:** For how to install and configure `cri-tools` with CRI runtimes like `containerd` or CRI-O, please also refer to other [howtos](./README.md). + +## Use `crictl` run Pods in Kata containers + +Sample config files in this document can be found [here](./data/crictl/). + +### Run `busybox` Pod + +#### Run pod sandbox with config file + +```bash +$ sudo crictl runp -r kata sandbox_config.json +16a62b035940f9c7d79fd53e93902d15ad21f7f9b3735f1ac9f51d16539b836b + +$ sudo crictl pods +POD ID CREATED STATE NAME NAMESPACE ATTEMPT +16a62b035940f 21 seconds ago Ready busybox-pod 0 +``` + +#### Create container in the pod sandbox with config file + +```bash +$ sudo crictl create 16a62b035940f container_config.json sandbox_config.json +e6ca0e0f7f532686236b8b1f549e4878e4fe32ea6b599a5d684faf168b429202 +``` + +List containers and check the container is in `Created` state: + +```bash +$ sudo crictl ps -a +CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID +e6ca0e0f7f532 docker.io/library/busybox:latest 19 seconds ago Created busybox-container 0 16a62b035940f +``` + +#### Start container + +```bash +$ sudo crictl start e6ca0e0f7f532 +e6ca0e0f7f532 +``` + +List containers and we can see that the container state has changed from `Created` to `Running`: + +```bash +$ sudo crictl ps +CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID +e6ca0e0f7f532 docker.io/library/busybox:latest About a minute ago Running busybox-container 0 16a62b035940f +``` + +And last we can `exec` into `busybox` container: + +```bash +$ sudo crictl exec -it e6ca0e0f7f532 sh +``` + +And run commands in it: + +``` +/ # hostname +busybox_host +/ # id +uid=0(root) gid=0(root) +``` + +### Run `redis` Pod + +In this example, we will create two Pods: one is for `redis` server, and another one is `redis` client. + +#### Create `redis-server` Pod + +It's also possible to start a container within a single command: + +```bash +$ sudo crictl run -r kata redis_server_container_config.json redis_server_sandbox_config.json +bb36e05c599125842c5193909c4de186b1cee3818f5d17b951b6a0422681ce4b +``` + +#### Create `redis-client` Pod + +```bash +$ sudo crictl run -r kata redis_client_container_config.json redis_client_sandbox_config.json +e344346c5414e3f51f97f20b2262e0b7afe457750e94dc0edb109b94622fc693 +``` + +After the new container started, we can check the running Pods and containers. + +```bash +$ sudo crictl pods +POD ID CREATED STATE NAME NAMESPACE ATTEMPT +469d08a7950e3 30 seconds ago Ready redis-client-pod 0 +02c12fdb08219 About a minute ago Ready redis-server-pod 0 + +$ sudo crictl ps +CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID +e344346c5414e docker.io/library/redis:6.0.8-alpine 35 seconds ago Running redis-client 0 469d08a7950e3 +bb36e05c59912 docker.io/library/redis:6.0.8-alpine About a minute ago Running redis-server 0 02c12fdb08219 +``` + +#### Check `redis` server is working + +To connect to the `redis-server`. First we need to get the `redis-server`'s IP address. + +```bash + +$ server=$(sudo crictl inspectp 02c12fdb08219 | jq .status.network.ip | tr -d '"' ) +$ echo $server +172.19.0.118 +``` + +Launch `redis-cli` in the new Pod and connect server running at `172.19.0.118`. + +```bash +$ sudo crictl exec -it e344346c5414e redis-cli -h $server +172.19.0.118:6379> get test-key +(nil) +172.19.0.118:6379> set test-key test-value +OK +172.19.0.118:6379> get test-key +"test-value" +``` + +Then back to `redis-server`, check if the `test-key` is set in server. + +```bash +$ sudo crictl exec -it bb36e05c59912 redis-cli get test-key +"test-val" +``` + +Returned `test-val` is just set by `redis-cli` in `redis-client` Pod. From 0cfcbf79b8d403f7e193f217388b65205972dd57 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 10:02:08 +0800 Subject: [PATCH 036/124] docs: add namespace key to pod/container config files If no namespace field in config files, CRI-O will failed: setting pod sandbox name and id: cannot generate pod name without namespace Signed-off-by: bin liu --- docs/how-to/data/crictl/busybox/container_config.json | 3 ++- docs/how-to/data/crictl/busybox/sandbox_config.json | 3 ++- .../data/crictl/redis/redis_client_container_config.json | 3 ++- docs/how-to/data/crictl/redis/redis_client_sandbox_config.json | 3 ++- .../data/crictl/redis/redis_server_container_config.json | 3 ++- docs/how-to/data/crictl/redis/redis_server_sandbox_config.json | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/how-to/data/crictl/busybox/container_config.json b/docs/how-to/data/crictl/busybox/container_config.json index 5772496514..fcea05a736 100644 --- a/docs/how-to/data/crictl/busybox/container_config.json +++ b/docs/how-to/data/crictl/busybox/container_config.json @@ -1,6 +1,7 @@ { "metadata": { - "name": "busybox-container" + "name": "busybox-container", + "namespace": "test.kata" }, "image": { "image": "docker.io/library/busybox:latest" diff --git a/docs/how-to/data/crictl/busybox/sandbox_config.json b/docs/how-to/data/crictl/busybox/sandbox_config.json index 963db8633c..d722555f44 100644 --- a/docs/how-to/data/crictl/busybox/sandbox_config.json +++ b/docs/how-to/data/crictl/busybox/sandbox_config.json @@ -1,7 +1,8 @@ { "metadata": { "name": "busybox-pod", - "uid": "busybox-pod" + "uid": "busybox-pod", + "namespace": "test.kata" }, "hostname": "busybox_host", "log_directory": "", diff --git a/docs/how-to/data/crictl/redis/redis_client_container_config.json b/docs/how-to/data/crictl/redis/redis_client_container_config.json index 95c42248ec..adda736843 100644 --- a/docs/how-to/data/crictl/redis/redis_client_container_config.json +++ b/docs/how-to/data/crictl/redis/redis_client_container_config.json @@ -1,6 +1,7 @@ { "metadata": { - "name": "redis-client" + "name": "redis-client", + "namespace": "test.kata" }, "image": { "image": "docker.io/library/redis:6.0.8-alpine" diff --git a/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json b/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json index 6613d3c0d9..35ad7d3ac6 100644 --- a/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json +++ b/docs/how-to/data/crictl/redis/redis_client_sandbox_config.json @@ -1,7 +1,8 @@ { "metadata": { "name": "redis-client-pod", - "uid": "test-redis-client-pod" + "uid": "test-redis-client-pod", + "namespace": "test.kata" }, "hostname": "redis-client", "log_directory": "", diff --git a/docs/how-to/data/crictl/redis/redis_server_container_config.json b/docs/how-to/data/crictl/redis/redis_server_container_config.json index faf1f444d0..4ac20725ba 100644 --- a/docs/how-to/data/crictl/redis/redis_server_container_config.json +++ b/docs/how-to/data/crictl/redis/redis_server_container_config.json @@ -1,6 +1,7 @@ { "metadata": { - "name": "redis-server" + "name": "redis-server", + "namespace": "test.kata" }, "image": { "image": "docker.io/library/redis:6.0.8-alpine" diff --git a/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json b/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json index 29af2d133d..1346765d87 100644 --- a/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json +++ b/docs/how-to/data/crictl/redis/redis_server_sandbox_config.json @@ -1,7 +1,8 @@ { "metadata": { "name": "redis-server-pod", - "uid": "test-redis-server-pod" + "uid": "test-redis-server-pod", + "namespace": "test.kata" }, "hostname": "redis-server", "log_directory": "", From fb7e9b4f32c9c0b064eed0a8f903f971c7e982cc Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Sat, 10 Oct 2020 13:52:15 +0800 Subject: [PATCH 037/124] agent: fix aarch64 build aarch64 needs libgcc to resolve some non-builtin symbols. Fixes: #909 Signed-off-by: Jianyong Wu Signed-off-by: Peng Tao --- src/agent/Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/agent/Makefile b/src/agent/Makefile index 8b97439e8f..6a5e7aef2c 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -46,6 +46,13 @@ ifeq ($(ARCH), ppc64le) $(warning "WARNING: powerpc64le-unknown-linux-musl target is unavailable") endif + +EXTRA_RUSTFLAGS := +ifeq ($(ARCH), aarch64) + override EXTRA_RUSTFLAGS = -C link-arg=-lgcc + $(warning "WARNING: aarch64-musl needs extra symbols from libgcc") +endif + TRIPLE = $(ARCH)-unknown-linux-$(LIBC) TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET) @@ -106,10 +113,10 @@ default: $(TARGET) show-header $(TARGET): $(GENERATED_CODE) $(TARGET_PATH) $(TARGET_PATH): $(SOURCES) | show-summary - @RUSTFLAGS="--deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) optimize: $(SOURCES) | show-summary show-header - @RUSTFLAGS='-C link-arg=-s --deny-warnings' cargo build --target $(TRIPLE) --$(BUILD_TYPE) + @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) show-header: @printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)" From 38212ba6d8df3e646442a4442d5077253e2f0b5c Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Sat, 10 Oct 2020 14:25:55 +0800 Subject: [PATCH 038/124] packaging: apply qemu v5.1 stable fixes Qemu v5.1 was released with an affending commit 9b3a35ec82 (virtio: verify that legacy support is not accidentally on). As a result, it breaks commandline compatiblilities for old qemu users. Upstream qemu has fixed it but no release has been put out yet. Let's apply these fixes by hand for now. Refs: https://www.mail-archive.com/qemu-devel@nongnu.org/msg729556.html Signed-off-by: Peng Tao --- ...cy-support-check-on-machine-types-le.patch | 143 ++++++++++++++++++ ...ost-vsock-pci-force-virtio-version-1.patch | 73 +++++++++ ...ser-vsock-pci-force-virtio-version-1.patch | 57 +++++++ ...ost-vsock-ccw-force-virtio-version-1.patch | 52 +++++++ 4 files changed, 325 insertions(+) create mode 100644 tools/packaging/qemu/patches/5.1.x/0002-virtio-skip-legacy-support-check-on-machine-types-le.patch create mode 100644 tools/packaging/qemu/patches/5.1.x/0003-vhost-vsock-pci-force-virtio-version-1.patch create mode 100644 tools/packaging/qemu/patches/5.1.x/0004-vhost-user-vsock-pci-force-virtio-version-1.patch create mode 100644 tools/packaging/qemu/patches/5.1.x/0005-vhost-vsock-ccw-force-virtio-version-1.patch diff --git a/tools/packaging/qemu/patches/5.1.x/0002-virtio-skip-legacy-support-check-on-machine-types-le.patch b/tools/packaging/qemu/patches/5.1.x/0002-virtio-skip-legacy-support-check-on-machine-types-le.patch new file mode 100644 index 0000000000..8d4b09f17b --- /dev/null +++ b/tools/packaging/qemu/patches/5.1.x/0002-virtio-skip-legacy-support-check-on-machine-types-le.patch @@ -0,0 +1,143 @@ +From d55f518248f263bb8d0852f98e47102ea09d4f89 Mon Sep 17 00:00:00 2001 +From: Stefano Garzarella +Date: Mon, 21 Sep 2020 14:25:03 +0200 +Subject: [PATCH 1/4] virtio: skip legacy support check on machine types less + than 5.1 + +Commit 9b3a35ec82 ("virtio: verify that legacy support is not accidentally +on") added a check that returns an error if legacy support is on, but the +device does not support legacy. + +Unfortunately some devices were wrongly declared legacy capable even if +they were not (e.g vhost-vsock). + +To avoid migration issues, we add a virtio-device property +(x-disable-legacy-check) to skip the legacy error, printing a warning +instead, for machine types < 5.1. + +Cc: qemu-stable@nongnu.org +Fixes: 9b3a35ec82 ("virtio: verify that legacy support is not accidentally on") +Suggested-by: Dr. David Alan Gilbert +Suggested-by: Cornelia Huck +Reviewed-by: Cornelia Huck +Signed-off-by: Stefano Garzarella +Message-Id: <20200921122506.82515-2-sgarzare@redhat.com> +Reviewed-by: Michael S. Tsirkin +Signed-off-by: Michael S. Tsirkin +--- + hw/core/machine.c | 1 + + hw/s390x/virtio-ccw.c | 15 ++++++++++++--- + hw/virtio/virtio-pci.c | 14 ++++++++++++-- + hw/virtio/virtio.c | 7 +++++++ + include/hw/virtio/virtio.h | 2 ++ + 5 files changed, 34 insertions(+), 5 deletions(-) + +diff --git a/hw/core/machine.c b/hw/core/machine.c +index 9b02fb2..d7f4a0d 100644 +--- a/hw/core/machine.c ++++ b/hw/core/machine.c +@@ -44,6 +44,7 @@ GlobalProperty hw_compat_5_0[] = { + { "vmport", "x-signal-unsupported-cmd", "off" }, + { "vmport", "x-report-vmx-type", "off" }, + { "vmport", "x-cmds-v2", "off" }, ++ { "virtio-device", "x-disable-legacy-check", "true" }, + }; + const size_t hw_compat_5_0_len = G_N_ELEMENTS(hw_compat_5_0); + +diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c +index 8d140dc..4582e94 100644 +--- a/hw/s390x/virtio-ccw.c ++++ b/hw/s390x/virtio-ccw.c +@@ -1122,9 +1122,18 @@ static void virtio_ccw_device_plugged(DeviceState *d, Error **errp) + } + + if (!virtio_ccw_rev_max(dev) && !virtio_legacy_allowed(vdev)) { +- error_setg(errp, "Invalid value of property max_rev " +- "(is %d expected >= 1)", virtio_ccw_rev_max(dev)); +- return; ++ /* ++ * To avoid migration issues, we allow legacy mode when legacy ++ * check is disabled in the old machine types (< 5.1). ++ */ ++ if (virtio_legacy_check_disabled(vdev)) { ++ warn_report("device requires revision >= 1, but for backward " ++ "compatibility max_revision=0 is allowed"); ++ } else { ++ error_setg(errp, "Invalid value of property max_rev " ++ "(is %d expected >= 1)", virtio_ccw_rev_max(dev)); ++ return; ++ } + } + + if (virtio_get_num_queues(vdev) > VIRTIO_QUEUE_MAX) { +diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c +index 02790e3..36524a5 100644 +--- a/hw/virtio/virtio-pci.c ++++ b/hw/virtio/virtio-pci.c +@@ -1597,8 +1597,18 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp) + + if (legacy) { + if (!virtio_legacy_allowed(vdev)) { +- error_setg(errp, "device is modern-only, use disable-legacy=on"); +- return; ++ /* ++ * To avoid migration issues, we allow legacy mode when legacy ++ * check is disabled in the old machine types (< 5.1). ++ */ ++ if (virtio_legacy_check_disabled(vdev)) { ++ warn_report("device is modern-only, but for backward " ++ "compatibility legacy is allowed"); ++ } else { ++ error_setg(errp, ++ "device is modern-only, use disable-legacy=on"); ++ return; ++ } + } + if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { + error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by" +diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c +index 3a3d012..a2edb4f 100644 +--- a/hw/virtio/virtio.c ++++ b/hw/virtio/virtio.c +@@ -3304,6 +3304,11 @@ bool virtio_legacy_allowed(VirtIODevice *vdev) + } + } + ++bool virtio_legacy_check_disabled(VirtIODevice *vdev) ++{ ++ return vdev->disable_legacy_check; ++} ++ + hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) + { + return vdev->vq[n].vring.desc; +@@ -3713,6 +3718,8 @@ static Property virtio_properties[] = { + DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features), + DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true), + DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true), ++ DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice, ++ disable_legacy_check, false), + DEFINE_PROP_END_OF_LIST(), + }; + +diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h +index 28cf3b9..b7ece7a 100644 +--- a/include/hw/virtio/virtio.h ++++ b/include/hw/virtio/virtio.h +@@ -101,6 +101,7 @@ struct VirtIODevice + bool use_started; + bool started; + bool start_on_kick; /* when virtio 1.0 feature has not been negotiated */ ++ bool disable_legacy_check; + VMChangeStateEntry *vmstate; + char *bus_name; + uint8_t device_endian; +@@ -394,5 +395,6 @@ static inline bool virtio_device_disabled(VirtIODevice *vdev) + } + + bool virtio_legacy_allowed(VirtIODevice *vdev); ++bool virtio_legacy_check_disabled(VirtIODevice *vdev); + + #endif +-- +1.8.3.1 diff --git a/tools/packaging/qemu/patches/5.1.x/0003-vhost-vsock-pci-force-virtio-version-1.patch b/tools/packaging/qemu/patches/5.1.x/0003-vhost-vsock-pci-force-virtio-version-1.patch new file mode 100644 index 0000000000..3ff433036d --- /dev/null +++ b/tools/packaging/qemu/patches/5.1.x/0003-vhost-vsock-pci-force-virtio-version-1.patch @@ -0,0 +1,73 @@ +From 6209070503989cf4f28549f228989419d4f0b236 Mon Sep 17 00:00:00 2001 +From: Stefano Garzarella +Date: Mon, 21 Sep 2020 14:25:04 +0200 +Subject: [PATCH 2/4] vhost-vsock-pci: force virtio version 1 + +Commit 9b3a35ec82 ("virtio: verify that legacy support is not +accidentally on") added a safety check that requires to set +'disable-legacy=on' on vhost-vsock-pci device: + + $ ./qemu-system-x86_64 ... -device vhost-vsock-pci,guest-cid=5 + qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=5: + device is modern-only, use disable-legacy=on + +virtio-vsock was introduced after the release of VIRTIO 1.0 +specifications, so it should be 'modern-only'. +In addition Cornelia verified that forcing a legacy mode on +vhost-vsock-pci device using x86-64 host and s390x guest, so with +different endianness, produces strange behaviours. + +This patch forces virtio version 1 and removes the 'transitional_name' +property removing the need to specify 'disable-legacy=on' on +vhost-vsock-pci device. + +To avoid migration issues, we force virtio version 1 only when +legacy check is enabled in the new machine types (>= 5.1). + +As the transitional device name is not commonly used, we do not +provide compatibility handling for it. + +Cc: qemu-stable@nongnu.org +Reported-by: Qian Cai +Reported-by: Qinghua Cheng +Buglink: https://bugzilla.redhat.com/show_bug.cgi?id=1868449 +Suggested-by: Cornelia Huck +Reviewed-by: Cornelia Huck +Signed-off-by: Stefano Garzarella +Message-Id: <20200921122506.82515-3-sgarzare@redhat.com> +Reviewed-by: Michael S. Tsirkin +Signed-off-by: Michael S. Tsirkin +--- + hw/virtio/vhost-vsock-pci.c | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/hw/virtio/vhost-vsock-pci.c b/hw/virtio/vhost-vsock-pci.c +index e56067b..205da8d 100644 +--- a/hw/virtio/vhost-vsock-pci.c ++++ b/hw/virtio/vhost-vsock-pci.c +@@ -44,6 +44,15 @@ static void vhost_vsock_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) + { + VHostVSockPCI *dev = VHOST_VSOCK_PCI(vpci_dev); + DeviceState *vdev = DEVICE(&dev->vdev); ++ VirtIODevice *virtio_dev = VIRTIO_DEVICE(vdev); ++ ++ /* ++ * To avoid migration issues, we force virtio version 1 only when ++ * legacy check is enabled in the new machine types (>= 5.1). ++ */ ++ if (!virtio_legacy_check_disabled(virtio_dev)) { ++ virtio_pci_force_virtio_1(vpci_dev); ++ } + + qdev_realize(vdev, BUS(&vpci_dev->bus), errp); + } +@@ -73,7 +82,6 @@ static void vhost_vsock_pci_instance_init(Object *obj) + static const VirtioPCIDeviceTypeInfo vhost_vsock_pci_info = { + .base_name = TYPE_VHOST_VSOCK_PCI, + .generic_name = "vhost-vsock-pci", +- .transitional_name = "vhost-vsock-pci-transitional", + .non_transitional_name = "vhost-vsock-pci-non-transitional", + .instance_size = sizeof(VHostVSockPCI), + .instance_init = vhost_vsock_pci_instance_init, +-- +1.8.3.1 diff --git a/tools/packaging/qemu/patches/5.1.x/0004-vhost-user-vsock-pci-force-virtio-version-1.patch b/tools/packaging/qemu/patches/5.1.x/0004-vhost-user-vsock-pci-force-virtio-version-1.patch new file mode 100644 index 0000000000..60a4e17678 --- /dev/null +++ b/tools/packaging/qemu/patches/5.1.x/0004-vhost-user-vsock-pci-force-virtio-version-1.patch @@ -0,0 +1,57 @@ +From 27eda699f59d430c33fc054a36a17251992e70dc Mon Sep 17 00:00:00 2001 +From: Stefano Garzarella +Date: Mon, 21 Sep 2020 14:25:05 +0200 +Subject: [PATCH 3/4] vhost-user-vsock-pci: force virtio version 1 + +Commit 9b3a35ec82 ("virtio: verify that legacy support is not +accidentally on") added a safety check that requires to set +'disable-legacy=on' on vhost-user-vsock-pci device: + + $ ./qemu-system-x86_64 ... \ + -chardev socket,id=char0,reconnect=0,path=/tmp/vhost4.socket \ + -device vhost-user-vsock-pci,chardev=char0 + qemu-system-x86_64: -device vhost-user-vsock-pci,chardev=char0: + device is modern-only, use disable-legacy=on + +virtio-vsock was introduced after the release of VIRTIO 1.0 +specifications, so it should be 'modern-only'. + +This patch forces virtio version 1 and removes the 'transitional_name' +property, as done for vhost-vsock-pci, removing the need to specify +'disable-legacy=on' on vhost-user-vsock-pci device. + +Cc: qemu-stable@nongnu.org +Suggested-by: Cornelia Huck +Reviewed-by: Cornelia Huck +Signed-off-by: Stefano Garzarella +Message-Id: <20200921122506.82515-4-sgarzare@redhat.com> +Reviewed-by: Michael S. Tsirkin +Signed-off-by: Michael S. Tsirkin +--- + hw/virtio/vhost-user-vsock-pci.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/hw/virtio/vhost-user-vsock-pci.c b/hw/virtio/vhost-user-vsock-pci.c +index 763f899..72a9619 100644 +--- a/hw/virtio/vhost-user-vsock-pci.c ++++ b/hw/virtio/vhost-user-vsock-pci.c +@@ -41,6 +41,9 @@ static void vhost_user_vsock_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) + VHostUserVSockPCI *dev = VHOST_USER_VSOCK_PCI(vpci_dev); + DeviceState *vdev = DEVICE(&dev->vdev); + ++ /* unlike vhost-vsock, we do not need to care about pre-5.1 compat */ ++ virtio_pci_force_virtio_1(vpci_dev); ++ + qdev_realize(vdev, BUS(&vpci_dev->bus), errp); + } + +@@ -69,7 +72,6 @@ static void vhost_user_vsock_pci_instance_init(Object *obj) + static const VirtioPCIDeviceTypeInfo vhost_user_vsock_pci_info = { + .base_name = TYPE_VHOST_USER_VSOCK_PCI, + .generic_name = "vhost-user-vsock-pci", +- .transitional_name = "vhost-user-vsock-pci-transitional", + .non_transitional_name = "vhost-user-vsock-pci-non-transitional", + .instance_size = sizeof(VHostUserVSockPCI), + .instance_init = vhost_user_vsock_pci_instance_init, +-- +1.8.3.1 diff --git a/tools/packaging/qemu/patches/5.1.x/0005-vhost-vsock-ccw-force-virtio-version-1.patch b/tools/packaging/qemu/patches/5.1.x/0005-vhost-vsock-ccw-force-virtio-version-1.patch new file mode 100644 index 0000000000..d459ecaeee --- /dev/null +++ b/tools/packaging/qemu/patches/5.1.x/0005-vhost-vsock-ccw-force-virtio-version-1.patch @@ -0,0 +1,52 @@ +From a6704a34cf02add13964149e0de6453ae62bd9db Mon Sep 17 00:00:00 2001 +From: Stefano Garzarella +Date: Mon, 21 Sep 2020 14:25:06 +0200 +Subject: [PATCH 4/4] vhost-vsock-ccw: force virtio version 1 + +virtio-vsock was introduced after the release of VIRTIO 1.0 +specifications, so it should be 'modern-only'. + +This patch forces virtio version 1 as done for vhost-vsock-pci. + +To avoid migration issues, we force virtio version 1 only when +legacy check is enabled in the new machine types (>= 5.1). + +Cc: qemu-stable@nongnu.org +Suggested-by: Cornelia Huck +Reviewed-by: Cornelia Huck +Signed-off-by: Stefano Garzarella +Message-Id: <20200921122506.82515-5-sgarzare@redhat.com> +Reviewed-by: Michael S. Tsirkin +Signed-off-by: Michael S. Tsirkin +--- + hw/s390x/vhost-vsock-ccw.c | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/hw/s390x/vhost-vsock-ccw.c b/hw/s390x/vhost-vsock-ccw.c +index 0822ecc..246416a 100644 +--- a/hw/s390x/vhost-vsock-ccw.c ++++ b/hw/s390x/vhost-vsock-ccw.c +@@ -40,9 +40,21 @@ static void vhost_vsock_ccw_class_init(ObjectClass *klass, void *data) + static void vhost_vsock_ccw_instance_init(Object *obj) + { + VHostVSockCCWState *dev = VHOST_VSOCK_CCW(obj); ++ VirtioCcwDevice *ccw_dev = VIRTIO_CCW_DEVICE(obj); ++ VirtIODevice *virtio_dev; + + virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev), + TYPE_VHOST_VSOCK); ++ ++ virtio_dev = VIRTIO_DEVICE(&dev->vdev); ++ ++ /* ++ * To avoid migration issues, we force virtio version 1 only when ++ * legacy check is enabled in the new machine types (>= 5.1). ++ */ ++ if (!virtio_legacy_check_disabled(virtio_dev)) { ++ ccw_dev->force_revision_1 = true; ++ } + } + + static const TypeInfo vhost_vsock_ccw_info = { +-- +1.8.3.1 From 7b53041bad54d85411c6ed233adc848ad7580762 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Sat, 10 Oct 2020 15:40:04 +0800 Subject: [PATCH 039/124] packaging: fix missing cloud_hypervisor_repo It is needed in order to build from source. Fixes: #916 Signed-off-by: Peng Tao --- .../static-build/cloud-hypervisor/build-static-clh.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 55db4f173c..40d35f3886 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -13,8 +13,17 @@ kata_version="${kata_version:-}" source "${script_dir}/../../scripts/lib.sh" +cloud_hypervisor_repo="${cloud_hypervisor_repo:-}" cloud_hypervisor_version="${cloud_hypervisor_version:-}" +if [ -z "$cloud_hypervisor_repo" ]; then + info "Get cloud_hypervisor information from runtime versions.yaml" + cloud_hypervisor_url=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.url" "${kata_version}") + [ -n "$cloud_hypervisor_url" ] || die "failed to get cloud_hypervisor url" + cloud_hypervisor_repo="${cloud_hypervisor_url}.git" +fi +[ -n "$cloud_hypervisor_repo" ] || die "failed to get cloud_hypervisor repo" + [ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version" "${kata_version}") [ -n "$cloud_hypervisor_version" ] || die "failed to get cloud_hypervisor version" From 7bb3e562bc66b7ac2cc0577e6ce56036918cbb7f Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Sat, 10 Oct 2020 18:49:35 +0800 Subject: [PATCH 040/124] packaging: fix cloud-hypervisor binary path 1. ensure build-static-clh.sh puts cloud-hypervisor under ./cloud-hypervisor directory 2. install cloud-hypervisor/cloud-hypervisor binary Signed-off-by: Peng Tao --- tools/packaging/release/kata-deploy-binaries.sh | 2 +- .../static-build/cloud-hypervisor/build-static-clh.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/packaging/release/kata-deploy-binaries.sh b/tools/packaging/release/kata-deploy-binaries.sh index d60c95e7a2..a900f51bb9 100755 --- a/tools/packaging/release/kata-deploy-binaries.sh +++ b/tools/packaging/release/kata-deploy-binaries.sh @@ -175,7 +175,7 @@ install_clh() { kata_version="${kata_version}" "${pkg_root_dir}/static-build/cloud-hypervisor/build-static-clh.sh" info "Install static cloud-hypervisor" mkdir -p "${destdir}/opt/kata/bin/" - sudo install -D --owner root --group root --mode 0744 cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor" + sudo install -D --owner root --group root --mode 0744 cloud-hypervisor/cloud-hypervisor "${destdir}/opt/kata/bin/cloud-hypervisor" pushd "${destdir}" # create tarball for github release action tar -czvf ../kata-static-clh.tar.gz * diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index 40d35f3886..f213bced17 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -31,7 +31,9 @@ pull_clh_released_binary() { info "Download cloud-hypervisor version: ${cloud_hypervisor_version}" cloud_hypervisor_binary="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${cloud_hypervisor_version}/cloud-hypervisor-static" - curl --fail -L ${cloud_hypervisor_binary} -o cloud-hypervisor || return 1 + curl --fail -L ${cloud_hypervisor_binary} -o cloud-hypervisor-static || return 1 + mkdir -p cloud-hypervisor + mv -f cloud-hypervisor-static cloud-hypervisor/cloud-hypervisor } build_clh_from_source() { From efddcb4ab8a025ea58db385cbf66781e96becfff Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 19:55:05 +0800 Subject: [PATCH 041/124] agent: use a local fn to reduce duplicated codes The same codes used twices, aggregated into a function can reduce codes. Signed-off-by: bin liu --- src/agent/src/rpc.rs | 50 ++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index ed71402742..003092fbf1 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -208,18 +208,7 @@ impl agentService { let cid = req.container_id.clone(); let mut cmounts: Vec = vec![]; - if req.timeout == 0 { - let s = Arc::clone(&self.sandbox); - let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); - } - }; - - ctr.destroy()?; - + let mut remove_container_resources = |sandbox: &mut Sandbox| -> Result<()> { // Find the sandbox storage used by this container let mounts = sandbox.container_mounts.get(&cid); if mounts.is_some() { @@ -240,6 +229,22 @@ impl agentService { sandbox.container_mounts.remove(cid.as_str()); sandbox.containers.remove(cid.as_str()); + Ok(()) + }; + + if req.timeout == 0 { + let s = Arc::clone(&self.sandbox); + let mut sandbox = s.lock().unwrap(); + let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { + Some(cr) => cr, + None => { + return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); + } + }; + + ctr.destroy()?; + + remove_container_resources(&mut sandbox)?; return Ok(()); } @@ -275,26 +280,7 @@ impl agentService { let s = self.sandbox.clone(); let mut sandbox = s.lock().unwrap(); - // Find the sandbox storage used by this container - let mounts = sandbox.container_mounts.get(&cid); - if mounts.is_some() { - let mounts = mounts.unwrap(); - - remove_mounts(&mounts)?; - - for m in mounts.iter() { - if sandbox.storages.get(m).is_some() { - cmounts.push(m.to_string()); - } - } - } - - for m in cmounts.iter() { - sandbox.unset_and_remove_sandbox_storage(m)?; - } - - sandbox.container_mounts.remove(&cid); - sandbox.containers.remove(cid.as_str()); + remove_container_resources(&mut sandbox)?; Ok(()) } From fa546600ff24159837169844a655e8a98371aba6 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 20:10:16 +0800 Subject: [PATCH 042/124] agent: use no-named closure to reduce codes For simple closures, inline closures can save codes. Signed-off-by: bin liu --- src/agent/src/rpc.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 003092fbf1..6ee5c5da42 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -669,15 +669,10 @@ impl protocols::agent_ttrpc::AgentService for agentService { let out: String = String::from_utf8(output.stdout).unwrap(); let mut lines: Vec = out.split('\n').map(|v| v.to_string()).collect(); - let predicate = |v| { - if v == "PID" { - return true; - } else { - return false; - } - }; - - let pid_index = lines[0].split_whitespace().position(predicate).unwrap(); + let pid_index = lines[0] + .split_whitespace() + .position(|v| v == "PID") + .unwrap(); let mut result = String::new(); result.push_str(lines[0].as_str()); From d3a36fa06f5bd893c491613dcdbaf605220232b3 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 12:23:44 +0000 Subject: [PATCH 043/124] agent: delete unused field in agentService The code is for test, and not needed now. Signed-off-by: bin liu --- src/agent/src/rpc.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 6ee5c5da42..0e86781dec 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -76,7 +76,6 @@ macro_rules! sl { #[derive(Clone)] pub struct agentService { sandbox: Arc>, - test: u32, } impl agentService { @@ -1063,7 +1062,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::StartTracingRequest, ) -> ttrpc::Result { - info!(sl!(), "start_tracing {:?} self.test={}", req, self.test); + info!(sl!(), "start_tracing {:?}", req); Ok(Empty::new()) } fn stop_tracing( @@ -1498,10 +1497,8 @@ fn find_process<'a>( } pub fn start(s: Arc>, server_address: &str) -> ttrpc::Server { - let agent_service = Box::new(agentService { - sandbox: s, - test: 1, - }) as Box; + let agent_service = Box::new(agentService { sandbox: s }) + as Box; let agent_worker = Arc::new(agent_service); From 42c48f54ed460c87098c07da603c9e07e70466d7 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 12:37:34 +0000 Subject: [PATCH 044/124] agent: add blank lines between methods In rpc.rs, there are no blank lines between methods, this commit add blank lines for these methods. Signed-off-by: bin liu --- src/agent/src/rpc.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 0e86781dec..1e3492f208 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -100,7 +100,6 @@ impl agentService { // re-scan PCI bus // looking for hidden devices - rescan_pci_bus().context("Could not rescan PCI bus")?; // Some devices need some extra processing (the ones invoked with @@ -293,7 +292,6 @@ impl agentService { let s = self.sandbox.clone(); let mut sandbox = s.lock().unwrap(); - // ignore string_user, not sure what it is let process = if req.process.is_some() { req.process.as_ref().unwrap() } else { @@ -555,10 +553,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { ttrpc::Code::INTERNAL, e.to_string(), ))), - Ok(_) => { - info!(sl!(), "exec process!\n"); - Ok(Empty::new()) - } + Ok(_) => Ok(Empty::new()), } } @@ -575,6 +570,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(_) => Ok(Empty::new()), } } + fn exec_process( &self, _ctx: &ttrpc::TtrpcContext, @@ -588,6 +584,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(_) => Ok(Empty::new()), } } + fn signal_process( &self, _ctx: &ttrpc::TtrpcContext, @@ -601,6 +598,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(_) => Ok(Empty::new()), } } + fn wait_process( &self, _ctx: &ttrpc::TtrpcContext, @@ -614,6 +612,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) => Ok(resp), } } + fn list_processes( &self, _ctx: &ttrpc::TtrpcContext, @@ -700,6 +699,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { resp.process_list = Vec::from(result); Ok(resp) } + fn update_container( &self, _ctx: &ttrpc::TtrpcContext, @@ -739,6 +739,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) } + fn stats_container( &self, _ctx: &ttrpc::TtrpcContext, @@ -832,6 +833,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) => Ok(resp), } } + fn read_stdout( &self, _ctx: &ttrpc::TtrpcContext, @@ -845,6 +847,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) => Ok(resp), } } + fn read_stderr( &self, _ctx: &ttrpc::TtrpcContext, @@ -858,6 +861,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) => Ok(resp), } } + fn close_stdin( &self, _ctx: &ttrpc::TtrpcContext, @@ -965,6 +969,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(iface) } + fn update_routes( &self, _ctx: &ttrpc::TtrpcContext, @@ -1000,6 +1005,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(routes) } + fn list_interfaces( &self, _ctx: &ttrpc::TtrpcContext, @@ -1028,6 +1034,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(interface) } + fn list_routes( &self, _ctx: &ttrpc::TtrpcContext, @@ -1057,6 +1064,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(routes) } + fn start_tracing( &self, _ctx: &ttrpc::TtrpcContext, @@ -1065,6 +1073,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { info!(sl!(), "start_tracing {:?}", req); Ok(Empty::new()) } + fn stop_tracing( &self, _ctx: &ttrpc::TtrpcContext, @@ -1175,6 +1184,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn add_arp_neighbors( &self, _ctx: &ttrpc::TtrpcContext, @@ -1200,6 +1210,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn online_cpu_mem( &self, _ctx: &ttrpc::TtrpcContext, @@ -1217,6 +1228,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn reseed_random_dev( &self, _ctx: &ttrpc::TtrpcContext, @@ -1231,6 +1243,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn get_guest_details( &self, _ctx: &ttrpc::TtrpcContext, @@ -1259,6 +1272,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(resp) } + fn mem_hotplug_by_probe( &self, _ctx: &ttrpc::TtrpcContext, @@ -1273,6 +1287,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn set_guest_date_time( &self, _ctx: &ttrpc::TtrpcContext, @@ -1287,6 +1302,7 @@ impl protocols::agent_ttrpc::AgentService for agentService { Ok(Empty::new()) } + fn copy_file( &self, _ctx: &ttrpc::TtrpcContext, @@ -1362,6 +1378,7 @@ impl protocols::health_ttrpc::Health for healthService { Ok(resp) } + fn version( &self, _ctx: &ttrpc::TtrpcContext, From 9e064ba19222d191b4e1acc3d0452dd77a6f4447 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 15:11:46 +0800 Subject: [PATCH 045/124] agent: use macro to simplify parse_cmdline function in config.rs In function parse_cmdline there are some similar codes, if we want to add more commandline arguments, the code will grow too long. Use macro can reduce some codes with the same logic/processing. Fixes: #914 Signed-off-by: bin liu --- src/agent/src/config.rs | 120 +++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 46 deletions(-) diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 3efab76e4e..227601f0b0 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -40,6 +40,36 @@ pub struct agentConfig { pub unified_cgroup_hierarchy: bool, } +// parse_cmdline_param parse commandline parameters. +macro_rules! parse_cmdline_param { + // commandline flags, without func to parse the option values + ($param:ident, $key:ident, $field:expr) => { + if $param.eq(&$key) { + $field = true; + continue; + } + }; + // commandline options, with func to parse the option values + ($param:ident, $key:ident, $field:expr, $func:ident) => { + if $param.starts_with(format!("{}=", $key).as_str()) { + let val = $func($param)?; + $field = val; + continue; + } + }; + // commandline options, with func to parse the option values, and match func + // to valid the values + ($param:ident, $key:ident, $field:expr, $func:ident, $guard:expr) => { + if $param.starts_with(format!("{}=", $key).as_str()) { + let val = $func($param)?; + if $guard(val) { + $field = val; + } + continue; + } + }; +} + impl agentConfig { pub fn new() -> agentConfig { agentConfig { @@ -60,51 +90,49 @@ impl agentConfig { let params: Vec<&str> = cmdline.split_ascii_whitespace().collect(); for param in params.iter() { // parse cmdline flags - if param.eq(&DEBUG_CONSOLE_FLAG) { - self.debug_console = true; - } - - if param.eq(&DEV_MODE_FLAG) { - self.dev_mode = true; - } + parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, self.debug_console); + parse_cmdline_param!(param, DEV_MODE_FLAG, self.dev_mode); // parse cmdline options - if param.starts_with(format!("{}=", LOG_LEVEL_OPTION).as_str()) { - let level = get_log_level(param)?; - self.log_level = level; - } + parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level); - if param.starts_with(format!("{}=", HOTPLUG_TIMOUT_OPTION).as_str()) { - let hotplugTimeout = get_hotplug_timeout(param)?; - // ensure the timeout is a positive value - if hotplugTimeout.as_secs() > 0 { - self.hotplug_timeout = hotplugTimeout; - } - } + // ensure the timeout is a positive value + parse_cmdline_param!( + param, + HOTPLUG_TIMOUT_OPTION, + self.hotplug_timeout, + get_hotplug_timeout, + |hotplugTimeout: time::Duration| hotplugTimeout.as_secs() > 0 + ); - if param.starts_with(format!("{}=", DEBUG_CONSOLE_VPORT_OPTION).as_str()) { - let port = get_vsock_port(param)?; - if port > 0 { - self.debug_console_vport = port; - } - } + // vsock port should be positive values + parse_cmdline_param!( + param, + DEBUG_CONSOLE_VPORT_OPTION, + self.debug_console_vport, + get_vsock_port, + |port| port > 0 + ); + parse_cmdline_param!( + param, + LOG_VPORT_OPTION, + self.log_vport, + get_vsock_port, + |port| port > 0 + ); - if param.starts_with(format!("{}=", LOG_VPORT_OPTION).as_str()) { - let port = get_vsock_port(param)?; - if port > 0 { - self.log_vport = port; - } - } - - if param.starts_with(format!("{}=", CONTAINER_PIPE_SIZE_OPTION).as_str()) { - let container_pipe_size = get_container_pipe_size(param)?; - self.container_pipe_size = container_pipe_size - } - - if param.starts_with(format!("{}=", UNIFIED_CGROUP_HIERARCHY_OPTION).as_str()) { - let b = get_bool_value(param, false); - self.unified_cgroup_hierarchy = b; - } + parse_cmdline_param!( + param, + CONTAINER_PIPE_SIZE_OPTION, + self.container_pipe_size, + get_container_pipe_size + ); + parse_cmdline_param!( + param, + UNIFIED_CGROUP_HIERARCHY_OPTION, + self.unified_cgroup_hierarchy, + get_bool_value + ); } if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) { @@ -185,11 +213,11 @@ fn get_hotplug_timeout(param: &str) -> Result { Ok(time::Duration::from_secs(value.unwrap())) } -fn get_bool_value(param: &str, default: bool) -> bool { +fn get_bool_value(param: &str) -> Result { let fields: Vec<&str> = param.split("=").collect(); if fields.len() != 2 { - return default; + return Ok(false); } let v = fields[1]; @@ -197,20 +225,20 @@ fn get_bool_value(param: &str, default: bool) -> bool { // bool let t: std::result::Result = v.parse(); if t.is_ok() { - return t.unwrap(); + return Ok(t.unwrap()); } // integer let i: std::result::Result = v.parse(); if i.is_err() { - return default; + return Ok(false); } // only `0` returns false, otherwise returns true - match i.unwrap() { + Ok(match i.unwrap() { 0 => false, _ => true, - } + }) } fn get_container_pipe_size(param: &str) -> Result { From ab64780a0b16a8dba0a8b068e09bc8cd5663ede6 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 17:54:15 +0800 Subject: [PATCH 046/124] agent: update not accurate comments This commit includes: - update comments that not matched the function name - file path with doubled slash Fixes: #922 Signed-off-by: bin liu --- src/agent/src/mount.rs | 10 +++++----- src/agent/src/namespace.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index a85d5c3a55..8e988af146 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -125,7 +125,7 @@ lazy_static! { // type of storage driver. type StorageHandler = fn(&Logger, &Storage, Arc>) -> Result; -// StorageHandlerList lists the supported drivers. +// STORAGEHANDLERLIST lists the supported drivers. #[cfg_attr(rustfmt, rustfmt_skip)] lazy_static! { pub static ref STORAGEHANDLERLIST: HashMap<&'static str, StorageHandler> = { @@ -510,7 +510,7 @@ pub fn get_mount_fs_type(mount_point: &str) -> Result { get_mount_fs_type_from_file(PROC_MOUNTSTATS, mount_point) } -// get_mount_fs_type returns the FS type corresponding to the passed mount point and +// get_mount_fs_type_from_file returns the FS type corresponding to the passed mount point and // any error ecountered. pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Result { if mount_point == "" { @@ -643,7 +643,7 @@ pub fn cgroups_mount(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result< // Enable memory hierarchical account. // For more information see https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt - online_device("/sys/fs/cgroup/memory//memory.use_hierarchy")?; + online_device("/sys/fs/cgroup/memory/memory.use_hierarchy")?; Ok(()) } @@ -654,8 +654,8 @@ pub fn remove_mounts(mounts: &Vec) -> Result<()> { Ok(()) } -// ensureDestinationExists will recursively create a given mountpoint. If directories -// are created, their permissions are initialized to mountPerm +// ensure_destination_exists will recursively create a given mountpoint. If directories +// are created, their permissions are initialized to mountPerm(0755) fn ensure_destination_exists(destination: &str, fs_type: &str) -> Result<()> { let d = Path::new(destination); if !d.exists() { diff --git a/src/agent/src/namespace.rs b/src/agent/src/namespace.rs index 374a67d91e..892332b3d7 100644 --- a/src/agent/src/namespace.rs +++ b/src/agent/src/namespace.rs @@ -75,7 +75,7 @@ impl Namespace { self } - // setup_persistent_ns creates persistent namespace without switching to it. + // setup creates persistent namespace without switching to it. // Note, pid namespaces cannot be persisted. pub fn setup(mut self) -> Result { if let Err(err) = fs::create_dir_all(&self.persistent_ns_dir) { From f70892a5bb01aed8d06548ad9a4785d01721bf7a Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Oct 2020 22:22:54 +0800 Subject: [PATCH 047/124] agent: use chain of Result to avoid early return Use rust `Result`'s `or_else`/`and_then` can write clean codes. And can avoid early return by check wether the `Result` is `Ok` or `Err`. Signed-off-by: bin liu --- src/agent/src/config.rs | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 227601f0b0..b46ab140ac 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -222,22 +222,16 @@ fn get_bool_value(param: &str) -> Result { let v = fields[1]; - // bool - let t: std::result::Result = v.parse(); - if t.is_ok() { - return Ok(t.unwrap()); - } - - // integer - let i: std::result::Result = v.parse(); - if i.is_err() { - return Ok(false); - } - - // only `0` returns false, otherwise returns true - Ok(match i.unwrap() { - 0 => false, - _ => true, + // first try to parse as bool value + v.parse::().or_else(|_err1| { + // then try to parse as integer value + v.parse::().or_else(|_err2| Ok(0)).and_then(|v| { + // only `0` returns false, otherwise returns true + Ok(match v { + 0 => false, + _ => true, + }) + }) }) } From d5d9928f97073dff8387d01f0349ebba39d6f8b9 Mon Sep 17 00:00:00 2001 From: bin liu Date: Mon, 12 Oct 2020 09:57:39 +0800 Subject: [PATCH 048/124] rustjail: delete unused test code The auto generated test code is no meanings, delete it. Signed-off-by: bin liu --- src/agent/rustjail/src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index d044408474..63dc77046f 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -554,11 +554,6 @@ pub fn grpc_to_oci(grpc: &grpcSpec) -> ociSpec { #[cfg(test)] mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } - #[allow(unused_macros)] #[macro_export] macro_rules! skip_if_not_root { From c9497c88e415d3bc0461f6d6f326f5db55e85624 Mon Sep 17 00:00:00 2001 From: bin liu Date: Mon, 12 Oct 2020 10:18:33 +0800 Subject: [PATCH 049/124] rustjail: delete codes commented out There are some uses/codes/struct fields are commented out, and may not turn into un-comment these codes, so delete these comments. Signed-off-by: bin liu --- src/agent/rustjail/src/cgroups/mod.rs | 1 - src/agent/rustjail/src/configs/mod.rs | 125 ----------------------- src/agent/rustjail/src/container.rs | 53 +++------- src/agent/rustjail/src/mount.rs | 6 -- src/agent/rustjail/src/process.rs | 7 -- src/agent/rustjail/src/specconv.rs | 142 -------------------------- 6 files changed, 15 insertions(+), 319 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs index c99ef469ac..abcaccb673 100644 --- a/src/agent/rustjail/src/cgroups/mod.rs +++ b/src/agent/rustjail/src/cgroups/mod.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 // -// use crate::configs::{FreezerState, Config}; use anyhow::{anyhow, Result}; use oci::LinuxResources; use protocols::agent::CgroupStats; diff --git a/src/agent/rustjail/src/configs/mod.rs b/src/agent/rustjail/src/configs/mod.rs index dea1682d26..21cc8b7c4d 100644 --- a/src/agent/rustjail/src/configs/mod.rs +++ b/src/agent/rustjail/src/configs/mod.rs @@ -366,128 +366,3 @@ impl IfPrioMap { format!("{} {}", self.interface, self.priority) } } - -/* -impl Config { - fn new(opts: &CreateOpts) -> Result { - if opts.spec.is_none() { - return Err(ErrorKind::ErrorCode("invalid createopts!".into())); - } - - let root = unistd::getcwd().chain_err(|| "cannot getwd")?; - let root = root.as_path().canonicalize().chain_err(|| - "cannot resolve root into absolute path")?; - let mut root = root.into(); - let cwd = root.clone(); - - let spec = opts.spec.as_ref().unwrap(); - if spec.root.is_none() { - return Err(ErrorKind::ErrorCode("no root".into())); - } - - let rootfs = PathBuf::from(&spec.root.as_ref().unwrap().path); - if rootfs.is_relative() { - root = format!("{}/{}", root, rootfs.into()); - } - - // handle annotations - let mut label = spec.annotations - .iter() - .map(|(key, value)| format!("{}={}", key, value)).collect(); - label.push(format!("bundle={}", cwd)); - - let mut config = Config { - rootfs: root, - no_pivot_root: opts.no_pivot_root, - readonlyfs: spec.root.as_ref().unwrap().readonly, - hostname: spec.hostname.clone(), - labels: label, - no_new_keyring: opts.no_new_keyring, - rootless_euid: opts.rootless_euid, - rootless_cgroups: opts.rootless_cgroups, - }; - - config.mounts = Vec::new(); - for m in &spec.mounts { - config.mounts.push(Mount::new(&cwd, &m)?); - } - - config.devices = create_devices(&spec)?; - config.cgroups = Cgroups::new(&opts)?; - - if spec.linux.as_ref().is_none() { - return Err(ErrorKind::ErrorCode("no linux configuration".into())); - } - let linux = spec.linux.as_ref().unwrap(); - - let propagation = MOUNTPROPAGATIONMAPPING.get(linux.rootfs_propagation); - if propagation.is_none() { - Err(ErrorKind::ErrorCode("rootfs propagation not support".into())); - } - - config.root_propagation = propagation.unwrap(); - if config.no_pivot_root && (config.root_propagation & MSFlags::MSPRIVATE != 0) { - return Err(ErrorKind::ErrorCode("[r]private is not safe without pivot root".into())); - } - - // handle namespaces - let m: HashMap = HashMap::new(); - for ns in &linux.namespaces { - if NAMESPACEMAPPING.get(&ns.r#type.as_str()).is_none() { - return Err(ErrorKind::ErrorCode("namespace don't exist".into())); - } - - if m.get(&ns.r#type).is_some() { - return Err(ErrorKind::ErrorCode(format!("duplicate ns {}", ns.r#type))); - } - - m.insert(ns.r#type, ns.path); - } - - if m.contains_key(oci::NETWORKNAMESPACE) { - let path = m.get(oci::NETWORKNAMESPACE).unwrap(); - if path == "" { - config.networks = vec![Network { - r#type: "loopback", - }]; - } - } - - if m.contains_key(oci::USERNAMESPACE) { - setup_user_namespace(&spec, &mut config)?; - } - - config.namespaces = m.iter().map(|(key, value)| Namespace { - r#type: key, - path: value, - }).collect(); - config.mask_paths = linux.mask_paths; - config.readonly_path = linux.readonly_path; - config.mount_label = linux.mount_label; - config.sysctl = linux.sysctl; - config.seccomp = None; - config.intelrdt = None; - - if spec.process.is_some() { - let process = spec.process.as_ref().unwrap(); - config.oom_score_adj = process.oom_score_adj; - config.process_label = process.selinux_label.clone(); - if process.capabilities.as_ref().is_some() { - let cap = process.capabilities.as_ref().unwrap(); - config.capabilities = Some(Capabilities { - ..cap - }) - } - } - config.hooks = None; - config.version = spec.version; - Ok(config) - } -} - - -impl Mount { - fn new(cwd: &str, m: &oci::Mount) -> Result { - } -} -*/ diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index c6d14e63cd..7db0440796 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -3,35 +3,32 @@ // SPDX-License-Identifier: Apache-2.0 // +use anyhow::{anyhow, bail, Context, Result}; use dirs; use lazy_static; +use libc::pid_t; use oci::{Hook, Linux, LinuxNamespace, LinuxResources, POSIXRlimit, Spec}; +use oci::{LinuxDevice, LinuxIDMapping}; use serde_json; +use std::clone::Clone; use std::ffi::{CStr, CString}; use std::fmt; +use std::fmt::Display; use std::fs; use std::os::unix::io::RawFd; use std::path::{Path, PathBuf}; -use std::time::SystemTime; -// use crate::sync::Cond; -use anyhow::{anyhow, bail, Context, Result}; -use libc::pid_t; -use oci::{LinuxDevice, LinuxIDMapping}; -use std::clone::Clone; -use std::fmt::Display; use std::process::Command; +use std::time::SystemTime; use cgroups::freezer::FreezerState; -use crate::process::Process; -// use crate::intelrdt::Manager as RdtManager; -use crate::log_child; -use crate::specconv::CreateOpts; -use crate::sync::*; -// use crate::stats::Stats; use crate::capabilities::{self, CAPSMAP}; use crate::cgroups::fs::Manager as FsManager; use crate::cgroups::Manager; +use crate::log_child; +use crate::process::Process; +use crate::specconv::CreateOpts; +use crate::sync::*; use crate::{mount, validator}; use protocols::agent::StatsContainerResponse; @@ -225,11 +222,6 @@ pub struct BaseState { init_process_pid: i32, #[serde(default)] init_process_start: u64, - /* - #[serde(default)] - created: SystemTime, - config: Config, - */ } pub trait BaseContainer { @@ -291,12 +283,8 @@ pub struct SyncPC { } pub trait Container: BaseContainer { - // fn checkpoint(&self, opts: &CriuOpts) -> Result<()>; - // fn restore(&self, p: &Process, opts: &CriuOpts) -> Result<()>; fn pause(&mut self) -> Result<()>; fn resume(&mut self) -> Result<()>; - // fn notify_oom(&self) -> Result<(Sender, Receiver)>; - // fn notify_memory_pressure(&self, lvl: PressureLevel) -> Result<(Sender, Receiver)>; } impl Container for LinuxContainer { @@ -627,7 +615,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { fifofd = std::env::var(FIFO_FD)?.parse::().unwrap(); } - //cleanup the env inherited from parent + // cleanup the env inherited from parent for (key, _) in env::vars() { env::remove_var(key); } @@ -636,7 +624,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { for e in env.iter() { let v: Vec<&str> = e.splitn(2, "=").collect(); if v.len() != 2 { - //info!(logger, "incorrect env config!"); continue; } env::set_var(v[0], v[1]); @@ -780,7 +767,6 @@ impl BaseContainer for LinuxContainer { return Err(anyhow!("exec fifo exists")); } unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?; - // defer!(fs::remove_file(&fifo_file)?); fifofd = fcntl::open( fifo_file.as_str(), @@ -1089,8 +1075,6 @@ fn do_exec(args: &[String]) -> ! { let a: Vec<&CStr> = sa.iter().map(|s| s.as_c_str()).collect(); if let Err(e) = unistd::execvp(p.as_c_str(), a.as_slice()) { - // info!(logger, "execve failed!!!"); - // info!(logger, "binary: {:?}, args: {:?}, envs: {:?}", p, a, env); match e { nix::Error::Sys(errno) => { std::process::exit(errno as i32); @@ -1198,7 +1182,6 @@ fn join_namespaces( info!(logger, "wait child received oci spec"); - // child.try_wait()?; read_sync(prfd)?; info!(logger, "send oci process from parent to child"); @@ -1211,7 +1194,7 @@ fn join_namespaces( let cm_str = serde_json::to_string(cm)?; write_sync(pwfd, SYNC_DATA, cm_str.as_str())?; - //wait child setup user namespace + // wait child setup user namespace info!(logger, "wait child setup user namespace"); read_sync(prfd)?; @@ -1270,7 +1253,7 @@ fn join_namespaces( read_sync(prfd)?; info!(logger, "get ready to run poststart hook!"); - //run poststart hook + // run poststart hook if spec.hooks.is_some() { info!(logger, "poststart hook"); let hooks = spec.hooks.as_ref().unwrap(); @@ -1508,7 +1491,6 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { let args = h.args.clone(); let envs = h.env.clone(); let state = serde_json::to_string(st)?; - // state.push_str("\n"); let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?; defer!({ @@ -1528,9 +1510,6 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { info!(logger, "hook child: {} status: {}", child, status); - // let _ = wait::waitpid(_ch, - // Some(WaitPidFlag::WEXITED | WaitPidFlag::__WALL)); - if status != 0 { if status == -libc::ETIMEDOUT { return Err(anyhow!(nix::Error::from_errno(Errno::ETIMEDOUT))); @@ -1571,7 +1550,7 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { .spawn() .unwrap(); - //send out our pid + // send out our pid tx.send(child.id() as libc::pid_t).unwrap(); info!(logger, "hook grand: {}", child.id()); @@ -1590,7 +1569,7 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { .unwrap() .read_to_string(&mut out) .unwrap(); - info!(logger, "{}", out.as_str()); + info!(logger, "child stdout: {}", out.as_str()); match child.wait() { Ok(exit) => { let code: i32 = if exit.success() { @@ -1660,8 +1639,6 @@ fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> { SYNC_DATA, std::str::from_utf8(&status.to_be_bytes()).unwrap_or_default(), ); - // let _ = wait::waitpid(Pid::from_raw(pid), - // Some(WaitPidFlag::WEXITED | WaitPidFlag::__WALL)); std::process::exit(0); } } diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index b45ca97cea..f759456e7a 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -912,8 +912,6 @@ fn mask_path(path: &str) -> Result<()> { return Err(nix::Error::Sys(Errno::EINVAL).into()); } - //info!("{}", path); - match mount( Some("/dev/null"), path, @@ -929,7 +927,6 @@ fn mask_path(path: &str) -> Result<()> { } Err(e) => { - //info!("{}: {}", path, e.as_errno().unwrap().desc()); return Err(e.into()); } @@ -944,8 +941,6 @@ fn readonly_path(path: &str) -> Result<()> { return Err(nix::Error::Sys(Errno::EINVAL).into()); } - //info!("{}", path); - match mount( Some(&path[1..]), path, @@ -963,7 +958,6 @@ fn readonly_path(path: &str) -> Result<()> { } Err(e) => { - //info!("{}: {}", path, e.as_errno().unwrap().desc()); return Err(e.into()); } diff --git a/src/agent/rustjail/src/process.rs b/src/agent/rustjail/src/process.rs index f27c4cda02..daba20e5f5 100644 --- a/src/agent/rustjail/src/process.rs +++ b/src/agent/rustjail/src/process.rs @@ -3,16 +3,11 @@ // SPDX-License-Identifier: Apache-2.0 // -// use std::process::{Stdio, Command, ExitStatus}; use libc::pid_t; use std::fs::File; use std::os::unix::io::RawFd; use std::sync::mpsc::Sender; -// use crate::configs::{Capabilities, Rlimit}; -// use crate::cgroups::Manager as CgroupManager; -// use crate::intelrdt::Manager as RdtManager; - use nix::fcntl::{fcntl, FcntlArg, OFlag}; use nix::sys::signal::{self, Signal}; use nix::sys::wait::{self, WaitStatus}; @@ -31,8 +26,6 @@ pub struct Process { pub exit_pipe_r: Option, pub exit_pipe_w: Option, pub extra_files: Vec, - // pub caps: Capabilities, - // pub rlimits: Vec, pub term_master: Option, pub tty: bool, pub parent_stdin: Option, diff --git a/src/agent/rustjail/src/specconv.rs b/src/agent/rustjail/src/specconv.rs index ca67bee91e..b61e544a39 100644 --- a/src/agent/rustjail/src/specconv.rs +++ b/src/agent/rustjail/src/specconv.rs @@ -4,8 +4,6 @@ // use oci::Spec; -// use crate::configs::namespaces; -// use crate::configs::device::Device; #[derive(Debug)] pub struct CreateOpts { @@ -17,143 +15,3 @@ pub struct CreateOpts { pub rootless_euid: bool, pub rootless_cgroup: bool, } -/* -const WILDCARD: i32 = -1; - -lazy_static! { - static ref NAEMSPACEMAPPING: HashMap<&'static str, &'static str> = { - let mut m = HashMap::new(); - m.insert(oci::PIDNAMESPACE, namespaces::NEWPID); - m.insert(oci::NETWORKNAMESPACE, namespaces::NEWNET); - m.insert(oci::UTSNAMESPACE, namespaces::NEWUTS); - m.insert(oci::MOUNTNAMESPACE, namespaces::NEWNS); - m.insert(oci::IPCNAMESPACE, namespaces::NEWIPC); - m.insert(oci::USERNAMESPACE, namespaces::NEWUSER); - m.insert(oci::CGROUPNAMESPACE, namespaces::NEWCGROUP); - m - }; - - static ref MOUNTPROPAGATIONMAPPING: HashMap<&'static str, MsFlags> = { - let mut m = HashMap::new(); - m.insert("rprivate", MsFlags::MS_PRIVATE | MsFlags::MS_REC); - m.insert("private", MsFlags::MS_PRIVATE); - m.insert("rslave", MsFlags::MS_SLAVE | MsFlags::MS_REC); - m.insert("slave", MsFlags::MS_SLAVE); - m.insert("rshared", MsFlags::MS_SHARED | MsFlags::MS_REC); - m.insert("shared", MsFlags::MS_SHARED); - m.insert("runbindable", MsFlags::MS_UNBINDABLE | MsFlags::MS_REC); - m.insert("unbindable", MsFlags::MS_UNBINDABLE); - m - }; - - static ref ALLOWED_DEVICES: Vec = { - let mut m = Vec::new(); - m.push(Device { - r#type: 'c', - major: WILDCARD, - minor: WILDCARD, - permissions: "m", - allow: true, - }); - - m.push(Device { - r#type: 'b', - major: WILDCARD, - minor: WILDCARD, - permissions: "m", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: "/dev/null".to_string(), - major: 1, - minor: 3, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/random"), - major: 1, - minor: 8, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/full"), - major: 1, - minor: 7, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/tty"), - major: 5, - minor: 0, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/zero"), - major: 1, - minor: 5, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/urandom"), - major: 1, - minor: 9, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from("/dev/console"), - major: 5, - minor: 1, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from(""), - major: 136, - minor: WILDCARD, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from(""), - major: 5, - minor: 2, - permissions: "rwm", - allow: true, - }); - - m.push(Device { - r#type: 'c', - path: String::from(""), - major: 10, - minor: 200, - permissions: "rwm", - allow: true, - }); - m - }; -} -*/ From ebe5ad1386f2db00b4bd719fec7d45174d463607 Mon Sep 17 00:00:00 2001 From: bin liu Date: Mon, 12 Oct 2020 06:08:27 +0000 Subject: [PATCH 050/124] rustjail: use Iterator to manipulate vector elements Use Iterator can save codes, and make code more readable Signed-off-by: bin liu --- src/agent/rustjail/src/container.rs | 46 +++++++++++++---------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 7db0440796..377c8124a0 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -387,7 +387,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { let linux = spec.linux.as_ref().unwrap(); // get namespace vector to join/new - let nses = get_namespaces(&linux)?; + let nses = get_namespaces(&linux); let mut userns = false; let mut to_new = CloneFlags::empty(); @@ -1140,24 +1140,21 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result> { } fn is_userns_enabled(linux: &Linux) -> bool { - for ns in &linux.namespaces { - if ns.r#type == "user" && ns.path == "" { - return true; - } - } - - false + linux + .namespaces + .iter() + .any(|ns| ns.r#type == "user" && ns.path == "") } -fn get_namespaces(linux: &Linux) -> Result> { - let mut ns: Vec = Vec::new(); - for i in &linux.namespaces { - ns.push(LinuxNamespace { - r#type: i.r#type.clone(), - path: i.path.clone(), - }); - } - Ok(ns) +fn get_namespaces(linux: &Linux) -> Vec { + linux + .namespaces + .iter() + .map(|ns| LinuxNamespace { + r#type: ns.r#type.clone(), + path: ns.path.clone(), + }) + .collect() } fn join_namespaces( @@ -1270,15 +1267,12 @@ fn join_namespaces( } fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIDMapping]) -> Result<()> { - let mut data = String::new(); - for m in maps { - if m.size == 0 { - continue; - } - - let val = format!("{} {} {}\n", m.container_id, m.host_id, m.size); - data = data + &val; - } + let data = maps + .iter() + .filter(|m| m.size != 0) + .map(|m| format!("{} {} {}\n", m.container_id, m.host_id, m.size)) + .collect::>() + .join(""); info!(logger, "mapping: {}", data); if !data.is_empty() { From 23246662b2489da2119e692cda381013b2f12d07 Mon Sep 17 00:00:00 2001 From: bin liu Date: Mon, 12 Oct 2020 16:59:02 +0800 Subject: [PATCH 051/124] agent: use ok_or/map_err instead of match Sometimes `Option.or_or` and `Result.map_err` may be simpler than match statement. Especially in rpc.rs, there are many `ctr.get_process` and `sandbox.get_container` which are using `match`. Signed-off-by: bin liu --- src/agent/src/rpc.rs | 277 +++++++++++++++++-------------------------- 1 file changed, 109 insertions(+), 168 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 1e3492f208..e7933b3ad9 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -181,12 +181,9 @@ impl agentService { let mut s = sandbox.lock().unwrap(); let sid = s.id.clone(); - let ctr: &mut LinuxContainer = match s.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); - } - }; + let ctr = s + .get_container(&cid) + .ok_or(anyhow!("Invalid container id"))?; ctr.exec()?; @@ -233,12 +230,9 @@ impl agentService { if req.timeout == 0 { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(anyhow!("Invalid container id"))?; ctr.destroy()?; @@ -254,15 +248,14 @@ impl agentService { let handle = thread::spawn(move || { let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid2.as_str()) { - Some(cr) => cr, - None => { - return; - } - }; - - ctr.destroy().unwrap(); - tx.send(1).unwrap(); + let _ctr = sandbox + .get_container(&cid2) + .ok_or(anyhow!("Invalid container id")) + .and_then(|ctr| { + ctr.destroy().unwrap(); + tx.send(1).unwrap(); + Ok(ctr) + }); }); if let Err(_) = rx.recv_timeout(Duration::from_secs(req.timeout as u64)) { @@ -302,12 +295,9 @@ impl agentService { let ocip = rustjail::process_grpc_to_oci(process); let p = Process::new(&sl!(), &ocip, exec_id.as_str(), false, pipe_size)?; - let ctr = match sandbox.get_container(cid.as_str()) { - Some(v) => v, - None => { - return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(anyhow!("Invalid container id"))?; ctr.run(p)?; @@ -387,12 +377,9 @@ impl agentService { } let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(anyhow!("Invalid container id"))?; let mut p = match ctr.processes.get_mut(&pid) { Some(p) => p, @@ -626,15 +613,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INVALID_ARGUMENT, - "invalid container id".to_string(), - ))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + "invalid container id".to_string(), + )))?; let pids = ctr.processes().unwrap(); @@ -711,15 +695,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - "invalid container id".to_string(), - ))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + "invalid container id".to_string(), + )))?; let resp = Empty::new(); @@ -749,15 +730,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let ctr: &mut LinuxContainer = match sandbox.get_container(cid.as_str()) { - Some(cr) => cr, - None => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - "invalid container id".to_string(), - ))); - } - }; + let ctr = sandbox + .get_container(&cid) + .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + "invalid container id".to_string(), + )))?; match ctr.stats() { Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( @@ -776,22 +754,19 @@ impl protocols::agent_ttrpc::AgentService for agentService { let cid = req.get_container_id(); let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - if let Some(ctr) = sandbox.get_container(cid) { - match ctr.pause() { - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))) - } - Ok(_) => return Ok(Empty::new()), - } - }; - Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INVALID_ARGUMENT, - "invalid argument".to_string(), - ))) + let ctr = sandbox + .get_container(&cid) + .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + "invalid container id".to_string(), + )))?; + + ctr.pause().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; + + Ok(Empty::new()) } fn resume_container( @@ -802,22 +777,19 @@ impl protocols::agent_ttrpc::AgentService for agentService { let cid = req.get_container_id(); let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - if let Some(ctr) = sandbox.get_container(cid) { - match ctr.resume() { - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))) - } - Ok(_) => return Ok(Empty::new()), - } - }; - Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INVALID_ARGUMENT, - "invalid argument: ".to_string(), - ))) + let ctr = sandbox + .get_container(&cid) + .ok_or(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + "invalid container id".to_string(), + )))?; + + ctr.resume().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; + + Ok(Empty::new()) } fn write_stdin( @@ -957,15 +929,14 @@ impl protocols::agent_ttrpc::AgentService for agentService { let rtnl = sandbox.rtnl.as_mut().unwrap(); - let iface = match rtnl.update_interface(interface.as_ref().unwrap()) { - Ok(v) => v, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( + let iface = rtnl + .update_interface(interface.as_ref().unwrap()) + .map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( ttrpc::Code::INTERNAL, format!("update interface: {:?}", e), - ))); - } - }; + )) + })?; Ok(iface) } @@ -986,16 +957,15 @@ impl protocols::agent_ttrpc::AgentService for agentService { } let rtnl = sandbox.rtnl.as_mut().unwrap(); + // get current routes to return when error out - let crs = match rtnl.list_routes() { - Ok(routes) => routes, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - format!("update routes: {:?}", e), - ))); - } - }; + let crs = rtnl.list_routes().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INTERNAL, + format!("update routes: {:?}", e), + )) + })?; + let v = match rtnl.update_routes(rs.as_ref()) { Ok(value) => value, Err(_) => crs, @@ -1020,15 +990,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { } let rtnl = sandbox.rtnl.as_mut().unwrap(); - let v = match rtnl.list_interfaces() { - Ok(value) => value, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - format!("list interface: {:?}", e), - ))); - } - }; + let v = rtnl.list_interfaces().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INTERNAL, + format!("list interface: {:?}", e), + )) + })?; interface.set_Interfaces(RepeatedField::from_vec(v)); @@ -1050,15 +1017,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let rtnl = sandbox.rtnl.as_mut().unwrap(); - let v = match rtnl.list_routes() { - Ok(value) => value, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - format!("list routes: {:?}", e), - ))); - } - }; + let v = rtnl.list_routes().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INTERNAL, + format!("list routes: {:?}", e), + )) + })?; routes.set_Routes(RepeatedField::from_vec(v)); @@ -1111,26 +1075,14 @@ impl protocols::agent_ttrpc::AgentService for agentService { } for m in req.kernel_modules.iter() { - match load_kernel_module(m) { - Ok(_) => (), - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))) - } - } + let _ = load_kernel_module(m).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; } - match s.setup_shared_namespaces() { - Ok(_) => (), - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))) - } - } + s.setup_shared_namespaces().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; } match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()) { @@ -1201,12 +1153,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { let rtnl = sandbox.rtnl.as_mut().unwrap(); - if let Err(e) = rtnl.add_arp_neighbors(neighs.as_ref()) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + rtnl.add_arp_neighbors(neighs.as_ref()).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } @@ -1491,26 +1440,18 @@ fn find_process<'a>( eid: &'a str, init: bool, ) -> Result<&'a mut Process> { - let ctr = match sandbox.get_container(cid) { - Some(v) => v, - None => return Err(anyhow!("Invalid container id")), - }; + let ctr = sandbox + .get_container(cid) + .ok_or(anyhow!("Invalid container id"))?; if init || eid == "" { - let p = match ctr.processes.get_mut(&ctr.init_process_pid) { - Some(v) => v, - None => return Err(anyhow!("cannot find init process!")), - }; - - return Ok(p); + return ctr + .processes + .get_mut(&ctr.init_process_pid) + .ok_or(anyhow!("cannot find init process!")); } - let p = match ctr.get_process(eid) { - Ok(v) => v, - Err(_) => return Err(anyhow!("Invalid exec id")), - }; - - Ok(p) + ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id")) } pub fn start(s: Arc>, server_address: &str) -> ttrpc::Server { @@ -1554,10 +1495,10 @@ fn update_container_namespaces( spec: &mut Spec, sandbox_pidns: bool, ) -> Result<()> { - let linux = match spec.linux.as_mut() { - None => return Err(anyhow!("Spec didn't container linux field")), - Some(l) => l, - }; + let linux = spec + .linux + .as_mut() + .ok_or(anyhow!("Spec didn't container linux field"))?; let namespaces = linux.namespaces.as_mut_slice(); for namespace in namespaces.iter_mut() { From 97acaa8124778c1c50ebea76e464cfd70caaa4ee Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 15 Sep 2020 14:07:34 +0100 Subject: [PATCH 052/124] docs: Add containerd install guide Create a containerd installation guide and a new `kata-manager` script for 2.0 that automated the steps outlined in the guide. Also cleaned up and improved the installation documentation in various ways, the most significant being: - Added legacy install link for 1.x installs. - Official packages section: - Removed "Contact" column (since it was empty!) - Reworded "Versions" column to clarify the versions are a minimum (to reduce maintenance burden). - Add a column to show which installation methods receive automatic updates. - Modified order of installation options in table and document to de-emphasise automatic installation and promote official packages and snap more. - Removed sections no longer relevant for 2.0. Fixes: #738. Signed-off-by: James O. D. Hunt --- docs/install/README.md | 106 ++-- .../containerd/containerd-install.md | 128 ++++ .../installing-with-kata-doc-to-script.md | 47 -- docs/install/installing-with-kata-manager.md | 47 -- utils/README.md | 55 ++ utils/kata-manager.sh | 580 ++++++++++++++++++ 6 files changed, 809 insertions(+), 154 deletions(-) create mode 100644 docs/install/container-manager/containerd/containerd-install.md delete mode 100644 docs/install/installing-with-kata-doc-to-script.md delete mode 100644 docs/install/installing-with-kata-manager.md create mode 100644 utils/README.md create mode 100755 utils/kata-manager.sh diff --git a/docs/install/README.md b/docs/install/README.md index d876b7c055..7831a36a87 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -1,98 +1,82 @@ # Kata Containers installation user guides -- [Kata Containers installation user guides](#kata-containers-installation-user-guides) - - [Prerequisites](#prerequisites) - - [Packaged installation methods](#packaged-installation-methods) - - [Official packages](#official-packages) - - [Automatic Installation](#automatic-installation) - - [Snap Installation](#snap-installation) - - [Scripted Installation](#scripted-installation) - - [Manual Installation](#manual-installation) - - [Build from source installation](#build-from-source-installation) - - [Installing on a Cloud Service Platform](#installing-on-a-cloud-service-platform) - - [Further information](#further-information) +* [Kata Containers installation user guides](#kata-containers-installation-user-guides) + * [Prerequisites](#prerequisites) + * [Legacy installation](#legacy-installation) + * [Packaged installation methods](#packaged-installation-methods) + * [Official packages](#official-packages) + * [Snap Installation](#snap-installation) + * [Automatic Installation](#automatic-installation) + * [Manual Installation](#manual-installation) + * [Build from source installation](#build-from-source-installation) + * [Installing on a Cloud Service Platform](#installing-on-a-cloud-service-platform) + * [Further information](#further-information) The following is an overview of the different installation methods available. All of these methods equally result in a system configured to run Kata Containers. ## Prerequisites + Kata Containers requires nested virtualization or bare metal. See the -[hardware requirements](../../src/runtime/README.md#hardware-requirements) +[hardware requirements](/src/runtime/README.md#hardware-requirements) to see if your system is capable of running Kata Containers. +## Legacy installation + +If you wish to install a legacy 1.x version of Kata Containers, see +[the Kata Containers 1.x installation documentation](https://github.com/kata-containers/documentation/tree/master/install/). + ## Packaged installation methods > **Notes:** > > - Packaged installation methods uses your distribution's native package format (such as RPM or DEB). +> - You are strongly encouraged to choose an installation method that provides +> automatic updates, to ensure you benefit from security updates and bug fixes. -| Installation method | Description | Distributions supported | -|------------------------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------| -| [Automatic](#automatic-installation) |Run a single command to install a full system | | -| [Using snap](#snap-installation) |Easy to install and automatic updates |any distro that supports snapd | -| [Using official distro packages](#official-packages) |Kata packages provided by Linux distributions official repositories | | -| [Scripted](#scripted-installation) |Generates an installation script which will result in a working system when executed | | -| [Manual](#manual-installation) |Allows the user to read a brief document and execute the specified commands step-by-step | | +| Installation method | Description | Automatic updates | Use case | +|------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------| +| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes | Recommended for most users. | +| [Using snap](#snap-installation) | Easy to install | yes | Good alternative to official distro packages. | +| [Automatic](#automatic-installation) | Run a single command to install a full system | **No!** | For those wanting the latest release quickly. | +| [Manual](#manual-installation) | Follow a guide step-by-step to install a working system | **No!** | For those who want the latest release with more control. | +| [Build from source](#build-from-source-installation) | Build the software components manually | **No!** | Power users and developers only. | ### Official packages Kata packages are provided by official distribution repositories for: -| Distribution (link to packages) | Versions | Contacts | -| -------------------------------------------------------- | ------------------------------------------------------------------------------ | -------- | -| [CentOS](centos-installation-guide.md) | 8 | | -| [Fedora](fedora-installation-guide.md) | 32, Rawhide | | -| [SUSE Linux Enterprise (SLE)](sle-installation-guide.md) | SLE 15 SP1, 15 SP2 | | -| [openSUSE](opensuse-installation-guide.md) | [Leap 15.1](opensuse-leap-15.1-installation-guide.md)
Leap 15.2, Tumbleweed | | +| Distribution (link to installation guide) | Minimum versions | +|----------------------------------------------------------|--------------------------------------------------------------------------------| +| [CentOS](centos-installation-guide.md) | 8 | +| [Fedora](fedora-installation-guide.md) | 32, Rawhide | +| [openSUSE](opensuse-installation-guide.md) | [Leap 15.1](opensuse-leap-15.1-installation-guide.md)
Leap 15.2, Tumbleweed | +| [SUSE Linux Enterprise (SLE)](sle-installation-guide.md) | SLE 15 SP1, 15 SP2 | - -### Automatic Installation - -[Use `kata-manager`](installing-with-kata-manager.md) to automatically install Kata packages. +> **Note::** +> +> All users are encouraged to uses the official distribution versions of Kata +> Containers unless they understand the implications of alternative methods. ### Snap Installation +> **Note:** The snap installation is available for all distributions which support `snapd`. + [![Get it from the Snap Store](https://snapcraft.io/static/images/badges/en/snap-store-black.svg)](https://snapcraft.io/kata-containers) [Use snap](snap-installation-guide.md) to install Kata Containers from https://snapcraft.io. -### Scripted Installation -[Use `kata-doc-to-script`](installing-with-kata-doc-to-script.md) to generate installation scripts that can be reviewed before they are executed. +### Automatic Installation + +[Use `kata-manager`](/utils/README.md) to automatically install a working Kata Containers system. ### Manual Installation -Manual installation instructions are available for [these distributions](#packaged-installation-methods) and document how to: -1. Add the Kata Containers repository to your distro package manager, and import the packages signing key. -2. Install the Kata Containers packages. -3. Install a supported container manager. -4. Configure the container manager to use Kata Containers as the default OCI runtime. Or, for Kata Containers 1.5.0 or above, configure the - `io.containerd.kata.v2` to be the runtime shim (see [containerd runtime v2 (shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) - and [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md)). -> **Notes on upgrading**: -> - If you are installing Kata Containers on a system that already has Clear Containers or `runv` installed, -> first read [the upgrading document](../Upgrading.md). - -> **Notes on releases**: -> - [This download server](http://download.opensuse.org/repositories/home:/katacontainers:/releases:/) -> hosts the Kata Containers packages built by OBS for all the supported architectures. -> Packages are available for the latest and stable releases (more info [here](../Stable-Branch-Strategy.md)). -> -> - The following guides apply to the latest Kata Containers release -> (a.k.a. `master` release). -> -> - When choosing a stable release, replace all `master` occurrences in the URLs -> with a `stable-x.y` version available on the [download server](http://download.opensuse.org/repositories/home:/katacontainers:/releases:/). - -> **Notes on packages source verification**: -> - The Kata packages hosted on the download server are signed with GPG to ensure integrity and authenticity. -> -> - The public key used to sign packages is available [at this link](https://raw.githubusercontent.com/kata-containers/tests/master/data/rpm-signkey.pub); the fingerprint is `9FDC0CB6 3708CF80 3696E2DC D0B37B82 6063F3ED`. -> -> - Only trust the signing key and fingerprint listed in the previous bullet point. Do not disable GPG checks, -> otherwise packages source and authenticity is not guaranteed. +Follow the [containerd installation guide](container-manager/containerd/containerd-install.md). ## Build from source installation + > **Notes:** > > - Power users who decide to build from sources should be aware of the @@ -104,6 +88,7 @@ who are comfortable building software from source to use the latest component versions. This is not recommended for normal users. ## Installing on a Cloud Service Platform + * [Amazon Web Services (AWS)](aws-installation-guide.md) * [Google Compute Engine (GCE)](gce-installation-guide.md) * [Microsoft Azure](azure-installation-guide.md) @@ -111,6 +96,7 @@ versions. This is not recommended for normal users. * [VEXXHOST OpenStack Cloud](vexxhost-installation-guide.md) ## Further information + * The [upgrading document](../Upgrading.md). * The [developer guide](../Developer-Guide.md). * The [runtime documentation](../../src/runtime/README.md). diff --git a/docs/install/container-manager/containerd/containerd-install.md b/docs/install/container-manager/containerd/containerd-install.md new file mode 100644 index 0000000000..c9da0120f7 --- /dev/null +++ b/docs/install/container-manager/containerd/containerd-install.md @@ -0,0 +1,128 @@ +# Install Kata Containers with containerd + +> **Note:** +> +> - If Kata Containers and / or containerd are packaged by your distribution, +> we recommend you install these versions to ensure they are updated when +> new releases are available. + +> **Warning:** +> +> - These instructions install the **newest** versions of Kata Containers and +> containerd from binary release packages. These versions may not have been +> tested with your distribution version. +> +> - Since your package manager is not being used, it is **your** +> responsibility to ensure these packages are kept up-to-date when new +> versions are released. +> +> - If you decide to proceed and install a Kata Containers release, you can +> still check for the latest version of Kata Containers by running +> `kata-runtime kata-check --only-list-releases`. +> +> - These instructions will not work for Fedora 31 and higher since those +> distribution versions only support cgroups version 2 by default. However, +> Kata Containers currently requires cgroups version 1 (on the host side). See +> https://github.com/kata-containers/kata-containers/issues/927 for further +> details. + +## Install Kata Containers + +> **Note:** +> +> If your distribution packages Kata Containers, we recommend you install that +> version. If it does not, or you wish to perform a manual installation, +> continue with the steps below. + +- Download a release from: + + - https://github.com/kata-containers/kata-containers/releases + + Note that Kata Containers uses [semantic versioning](https://semver.org) so + you should install a version that does *not* include a dash ("-"), since this + indicates a pre-release version. + +- Unpack the downloaded archive. + + Kata Containers packages use a `/opt/kata/` prefix so either add that to + your `PATH`, or create symbolic links for the following commands. The + advantage of using symbolic links is that the `systemd(1)` configuration file + for containerd will not need to be modified to allow the daemon to find this + binary (see the [section on installing containerd](#install-containerd) below). + + | Command | Description | + |-|-| + | `/opt/kata/bin/containerd-shim-kata-v2` | The main Kata 2.x binary | + | `/opt/kata/bin/kata-collect-data.sh` | Data collection script used for [raising issues](https://github.com/kata-containers/kata-containers/issues) | + | `/opt/kata/bin/kata-runtime` | Utility command | + +- Check installation by showing version details: + + ```bash + $ kata-runtime --version + ``` + +## Install containerd + +> **Note:** +> +> If your distribution packages containerd, we recommend you install that +> version. If it does not, or you wish to perform a manual installation, +> continue with the steps below. + +- Download a release from: + + - https://github.com/containerd/containerd/releases + +- Unpack the downloaded archive. + +- Configure containerd + + - Download the standard `systemd(1)` service file and install to + `/etc/systemd/system/`: + + - https://raw.githubusercontent.com/containerd/containerd/master/containerd.service + + > **Notes:** + > + > - You will need to reload the systemd configuration after installing this + > file. + > + > - If you have not created a symbolic link for + > `/opt/kata/bin/containerd-shim-kata-v2`, you will need to modify this + > file to ensure the containerd daemon's `PATH` contains `/opt/kata/`. + > See the `Environment=` command in `systemd.exec(5)` for further + > details. + + - Add the Kata Containers configuration to the containerd configuration file: + + ```toml + [plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "kata" + + [plugins.cri.containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" + ``` + + > **Note:** + > + > The containerd daemon needs to be able to find the + > `containerd-shim-kata-v2` binary to allow Kata Containers to be created. + + - Start the containerd service. + +## Test the installation + +You are now ready to run Kata Containers. You can perform a simple test by +running the following commands: + +```bash +$ image="docker.io/library/busybox:latest" +$ sudo ctr image pull "$image" +$ sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "$image" test-kata uname -r +``` + +The last command above shows details of the kernel version running inside the +container, which will likely be different to the host kernel version. diff --git a/docs/install/installing-with-kata-doc-to-script.md b/docs/install/installing-with-kata-doc-to-script.md deleted file mode 100644 index 3f531f02f1..0000000000 --- a/docs/install/installing-with-kata-doc-to-script.md +++ /dev/null @@ -1,47 +0,0 @@ -# Installing with `kata-doc-to-script` - -* [Introduction](#introduction) -* [Packages Installation](#packages-installation) -* [Docker Installation and Setup](#docker-installation-and-setup) - -## Introduction -Use [these installation instructions](README.md#packaged-installation-methods) together with -[`kata-doc-to-script`](https://github.com/kata-containers/tests/blob/master/.ci/kata-doc-to-script.sh) -to generate installation bash scripts. - -> Note: -> - Only the Docker container manager installation can be scripted. For other setups you must -> install and configure the container manager manually. - -## Packages Installation - -```bash -$ source /etc/os-release -$ curl -fsSL -O https://raw.githubusercontent.com/kata-containers/documentation/master/install/${ID}-installation-guide.md -$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/.ci/kata-doc-to-script.sh) ${ID}-installation-guide.md ${ID}-install.sh" -``` - -For example, if your distribution is CentOS, the previous example will generate a runnable shell script called `centos-install.sh`. -To proceed with the installation, run: - -```bash -$ source /etc/os-release -$ bash "./${ID}-install.sh" -``` - -## Docker Installation and Setup - -```bash -$ source /etc/os-release -$ curl -fsSL -O https://raw.githubusercontent.com/kata-containers/documentation/master/install/docker/${ID}-docker-install.md -$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/.ci/kata-doc-to-script.sh) ${ID}-docker-install.md ${ID}-docker-install.sh" -``` - -For example, if your distribution is CentOS, this will generate a runnable shell script called `centos-docker-install.sh`. - -To proceed with the Docker installation, run: - -```bash -$ source /etc/os-release -$ bash "./${ID}-docker-install.sh" -``` diff --git a/docs/install/installing-with-kata-manager.md b/docs/install/installing-with-kata-manager.md deleted file mode 100644 index fffa3bbf9a..0000000000 --- a/docs/install/installing-with-kata-manager.md +++ /dev/null @@ -1,47 +0,0 @@ -# Installing with `kata-manager` - -* [Introduction](#introduction) -* [Full Installation](#full-installation) -* [Install the Kata packages only](#install-the-kata-packages-only) -* [Further Information](#further-information) - -## Introduction -`kata-manager` automates the Kata Containers installation procedure documented for [these Linux distributions](README.md#packaged-installation-methods). - -> **Note**: -> - `kata-manager` requires `curl` and `sudo` installed on your system. -> -> - Full installation mode is only available for Docker container manager. For other setups, you -> can still use `kata-manager` to [install Kata package](#install-the-kata-packages-only), and then setup your container manager manually. -> -> - You can run `kata-manager` in dry run mode by passing the `-n` flag. Dry run mode allows you to review the -> commands that `kata-manager` would run, without doing any change to your system. - - -## Full Installation -This command does the following: -1. Installs Kata Containers packages -2. Installs Docker -3. Configure Docker to use the Kata OCI runtime by default - -```bash -$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-docker-system" -``` - - -## Install the Kata packages only -Use the following command to only install Kata Containers packages. - -```bash -$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/tests/master/cmd/kata-manager/kata-manager.sh) install-packages" -``` - -## Further Information -For more information on what `kata-manager` can do, refer to the [`kata-manager` page](https://github.com/kata-containers/tests/blob/master/cmd/kata-manager). diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000000..4294d31f0a --- /dev/null +++ b/utils/README.md @@ -0,0 +1,55 @@ +# Utilities + +# Kata Manager + +> **Warning:** +> +> - Kata Manager will not work for Fedora 31 and higher since those +> distribution versions only support cgroups version 2 by default. However, +> Kata Containers currently requires cgroups version 1 (on the host side). See +> https://github.com/kata-containers/kata-containers/issues/927 for further +> details. + +> **Note:** +> +> We recommend users install Kata Containers using +> [official distribution packages](../docs/install/README.md#official-packages), where available. + +The [`kata-manager.sh`](kata-manager.sh) script automatically installs and +configures Kata Containers and containerd. + +This scripted method installs the latest versions of Kata Containers and +containerd. However, be aware of the following before proceeding: + +- Packages will **not** be automatically updated + + Since a package manager is not being used, it is **your** responsibility + to ensure these packages are kept up-to-date when new versions are released + to ensure you are using a version that includes the latest security and bug fixes. + +- Potentially untested versions or version combinations + + This script installs the *newest* versions of Kata Containers + and containerd from binary release packages. These versions may + not have been tested with your distribution version. + +If you still wish to continue, but prefer a manual installation, see +[the containerd installation guide](/docs/install/container-manager/containerd/containerd-install.md). + +## Install a minimal Kata Containers system + +To install and configure a system with Kata Containers and containerd, run: + +```bash +$ bash -c "$(curl -fsSL https://raw.githubusercontent.com/kata-containers/kata-containers/2.0-dev/utils/kata-manager.sh)" +``` + +> **Notes:** +> +> - The script must be run on a system that does not have Kata Containers or +> containerd already installed on it. +> +> - The script accepts up to two parameters which can be used to test +> pre-release versions (a Kata Containers version, and a containerd +> version). If either version is unspecified or specified as `""`, the +> latest official version will be installed. diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh new file mode 100755 index 0000000000..1538eb8a60 --- /dev/null +++ b/utils/kata-manager.sh @@ -0,0 +1,580 @@ +#!/bin/bash +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +[ -n "${DEBUG:-}" ] && set -o xtrace + +readonly script_name=${0##*/} + +readonly kata_project="Kata Containers" +readonly containerd_project="containerd" + +readonly kata_slug="kata-containers/kata-containers" +readonly containerd_slug="containerd/containerd" + +readonly kata_project_url="https://github.com/${kata_slug}" +readonly containerd_project_url="https://github.com/${containerd_slug}" + +readonly kata_releases_url="https://api.github.com/repos/${kata_slug}/releases" +readonly containerd_releases_url="https://api.github.com/repos/${containerd_slug}/releases" + +# Directory created when unpacking a binary release archive downloaded from +# $kata_releases_url. +readonly kata_install_dir="${kata_install_dir:-/opt/kata}" + +# containerd shim v2 details +readonly kata_runtime_name="kata" +readonly kata_runtime_type="io.containerd.${kata_runtime_name}.v2" +readonly kata_shim_v2="containerd-shim-${kata_runtime_name}-v2" + +# Systemd unit name for containerd daemon +readonly containerd_service_name="containerd.service" + +# Directory in which to create symbolic links +readonly link_dir=${link_dir:-/usr/bin} + +readonly tmpdir=$(mktemp -d) + +readonly warnings=$(cat <&2 "ERROR: $*" + exit 1 +} + +info() +{ + echo -e "INFO: $*" +} + +cleanup() +{ + [ -d "$tmpdir" ] && rm -rf "$tmpdir" +} + +# Determine the latest GitHub release for a project. +# +# Parameter: GitHub API URL for a projects releases. +# Note: Releases are assumed to use semver (https://semver.org) version +# numbers. +github_get_latest_release() +{ + local url="${1:-}" + + # Notes: + # + # - The sort(1) call; none of the standard utilities support semver + # so attempt to perform a semver sort manually. + # - Pre-releases are excluded via the select() call. + local latest=$(curl -sL "$url" |\ + jq -r '.[].tag_name | select(contains("-") | not)' |\ + sort -t "." -k1,1n -k2,2n -k3,3n |\ + tail -1 || true) + + [ -z "$latest" ] && die "Cannot determine latest release from $url" + + echo "$latest" +} + +# Returns the actual version to download based on the specified one; if the +# specified version is blank, return the latest version. +github_resolve_version_to_download() +{ + local url="${1:-}" + local requested_version="${2:-}" + + local version="" + + if [ -n "$requested_version" ] + then + version="$requested_version" + else + version=$(github_get_latest_release "$url" || true) + fi + + echo "$version" +} + +# Return the GitHub URL for a particular release's pre-built binary package. +# +# Parameters: +# +# - GitHub API URL for a project's releases. +# - Release version to look for. +github_get_release_file_url() +{ + local url="${1:-}" + local version="${2:-}" + + download_url=$(curl -sL "$url" |\ + jq --arg version "$version" \ + -r '.[] | select(.tag_name == $version) | .assets[0].browser_download_url' || true) + + [ "$download_url" = null ] && download_url="" + [ -z "$download_url" ] && die "Cannot determine download URL for version $version ($url)" + + local arch=$(uname -m) + + [ "$arch" = x86_64 ] && arch="($arch|amd64)" + echo "$download_url" | egrep -q "$arch" || die "No release for '$arch architecture ($url)" + + echo "$download_url" +} + +# Download the release file for the specified projects release version and +# return the full path to the downloaded file. +# +# Parameters: +# +# - GitHub API URL for a project's releases. +# - Release version to download. +github_download_release() +{ + local url="${1:-}" + local version="${2:-}" + + pushd "$tmpdir" >/dev/null + + local download_url=$(github_get_release_file_url \ + "$url" \ + "$version" || true) + + [ -z "$download_url" ] && \ + die "Cannot determine download URL for version $version" + + # Don't specify quiet mode here so the user can observe download + # progress. + curl -LO "$download_url" + + local filename=$(echo "$download_url" | awk -F'/' '{print $NF}') + + ls -d "${PWD}/${filename}" + + popd >/dev/null +} + +usage() +{ + cat < []] + +Description: Install $kata_project [1] and $containerd_project [2] from GitHub release binaries. + +Options: + + -h : Show this help statement. + +Notes: + +- The version strings must refer to official GitHub releases for each project. + If not specified or set to "", install the latest available version. + +See also: + +[1] - $kata_project_url +[2] - $containerd_project_url + +$warnings + +Advice: + +- You can check the latest version of Kata Containers by running: + + $ kata-runtime kata-check --only-list-releases + +EOT +} + +# Determine if the system only supports cgroups v2. +# +# - Writes "true" to stdout if only cgroups v2 are supported. +# - Writes "false" to stdout if cgroups v1 or v1+v2 are available. +# - Writes a blank string to stdout if cgroups are not available. +only_supports_cgroups_v2() +{ + local v1=$(mount|awk '$5 ~ /^cgroup$/ { print; }' || true) + local v2=$(mount|awk '$5 ~ /^cgroup2$/ { print; }' || true) + + [ -n "$v1" ] && [ -n "$v2" ] && { echo "false"; return 0; } || true + [ -n "$v1" ] && { echo "false"; return 0; } || true + [ -n "$v2" ] && { echo "true"; return 0; } || true + + return 0 +} + +pre_checks() +{ + info "Running pre-checks" + + command -v "${kata_shim_v2}" &>/dev/null \ + && die "Please remove existing $kata_project installation" + + command -v containerd &>/dev/null \ + && die "$containerd_project already installed" + + systemctl list-unit-files --type service |\ + egrep -q "^${containerd_service_name}\>" \ + && die "$containerd_project already installed" + + local cgroups_v2_only=$(only_supports_cgroups_v2 || true) + + local url="https://github.com/kata-containers/kata-containers/issues/927" + + [ "$cgroups_v2_only" = "true" ] && \ + die "$kata_project does not yet fully support cgroups v2 - see $url" + + return 0 +} + +check_deps() +{ + info "Checking dependencies" + + # Maps command names to package names using a colon delimiter + local elems=() + + elems+=("curl:curl") + elems+=("git:git") + elems+=("jq:jq") + elems+=("tar:tar") + + local pkgs_to_install=() + + local elem + + for elem in "${elems[@]}" + do + local cmd=$(echo "$elem"|cut -d: -f1) + local pkg=$(echo "$elem"|cut -d: -f2-) + + command -v "$cmd" &>/dev/null && continue + + pkgs_to_install+=("$pkg") + done + + [ "${#pkgs_to_install[@]}" -eq 0 ] && return 0 + + local packages="${pkgs_to_install[@]}" + + info "Installing packages '$packages'" + + case "$ID" in + centos|rhel) sudo yum -y install $packages ;; + debian|ubuntu) sudo apt-get -y install $packages ;; + fedora) sudo dnf -y install $packages ;; + opensuse*|sles) sudo zypper install -y $packages ;; + *) die "Unsupported distro: $ID" + esac +} + +setup() +{ + trap cleanup EXIT + source /etc/os-release || source /usr/lib/os-release + + pre_checks + check_deps +} + +# Download the requested version of the specified project. +# +# Returns the resolve version number and the full path to the downloaded file +# separated by a colon. +github_download_package() +{ + local releases_url="${1:-}" + local requested_version="${2:-}" + local project="${3:-}" + + [ -z "$releases_url" ] && die "need releases URL" + [ -z "$project" ] && die "need project URL" + + local version=$(github_resolve_version_to_download \ + "$releases_url" \ + "$version" || true) + + [ -z "$version" ] && die "Unable to determine $project version to download" + + local file=$(github_download_release \ + "$releases_url" \ + "$version") + + echo "${version}:${file}" +} + +install_containerd() +{ + local requested_version="${1:-}" + + local project="$containerd_project" + + local version_desc="latest version" + [ -n "$requested_version" ] && version_desc="version $requested_version" + + info "Downloading $project release ($version_desc)" + + local results=$(github_download_package \ + "$containerd_releases_url" \ + "$requested_version" \ + "$project") + + [ -z "$results" ] && die "Cannot download $project release file" + + local version=$(echo "$results"|cut -d: -f1) + local file=$(echo "$results"|cut -d: -f2-) + + [ -z "$version" ] && die "Cannot determine $project resolved version" + [ -z "$file" ] && die "Cannot determine $project release file" + + info "Installing $project release $version from $file" + + sudo tar -C /usr/local -xvf "${file}" + + sudo ln -s /usr/local/bin/ctr "${link_dir}" + + info "$project installed\n" +} + +configure_containerd() +{ + local project="$containerd_project" + + info "Configuring $project" + + local cfg="/etc/containerd/config.toml" + + pushd "$tmpdir" >/dev/null + + local service_url=$(printf "%s/%s/%s/%s" \ + "https://raw.githubusercontent.com" \ + "${containerd_slug}" \ + "master" \ + "${containerd_service_name}") + + curl -LO "$service_url" + + printf "# %s: Service installed for Kata Containers\n" \ + "$(date -Iseconds)" |\ + tee -a "$containerd_service_name" + + local systemd_unit_dir="/etc/systemd/system" + sudo mkdir -p "$systemd_unit_dir" + + local dest="${systemd_unit_dir}/${containerd_service_name}" + + sudo cp "${containerd_service_name}" "${dest}" + sudo systemctl daemon-reload + + info "Installed ${dest}" + + popd >/dev/null + + # Backup the original containerd configuration: + sudo mkdir -p "$(dirname $cfg)" + + sudo test -e "$cfg" || { + sudo touch "$cfg" + info "Created $cfg" + } + + local original="${cfg}-pre-kata-$(date -I)" + + sudo grep -q "$kata_runtime_type" "$cfg" || { + sudo cp "$cfg" "${original}" + info "Backed up $cfg to $original" + } + + # Add the Kata Containers configuration details: + + sudo grep -q "$kata_runtime_type" "$cfg" || { + cat <<-EOT | sudo tee -a "$cfg" + [plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "${kata_runtime_name}" + [plugins.cri.containerd.runtimes.${kata_runtime_name}] + runtime_type = "${kata_runtime_type}" +EOT + + info "Modified $cfg" + } + + sudo systemctl start containerd + + info "Configured $project\n" +} + +install_kata() +{ + local requested_version="${1:-}" + + local project="$kata_project" + + local version_desc="latest version" + [ -n "$requested_version" ] && version_desc="version $requested_version" + + info "Downloading $project release ($version_desc)" + + local results=$(github_download_package \ + "$kata_releases_url" \ + "$requested_version" \ + "$project") + + [ -z "$results" ] && die "Cannot download $project release file" + + local version=$(echo "$results"|cut -d: -f1) + local file=$(echo "$results"|cut -d: -f2-) + + [ -z "$version" ] && die "Cannot determine $project resolved version" + [ -z "$file" ] && die "Cannot determine $project release file" + + # Allow the containerd service to find the Kata shim and users to find + # important commands: + local create_links_for=() + + create_links_for+=("$kata_shim_v2") + create_links_for+=("kata-collect-data.sh") + create_links_for+=("kata-runtime") + + local from_dir=$(printf "%s/bin" "$kata_install_dir") + + # Since we're unpacking to the root directory, perform a sanity check + # on the archive first. + local unexpected=$(tar -tf "${file}" |\ + egrep -v "^(\./opt/$|\.${kata_install_dir}/)" || true) + + [ -n "$unexpected" ] && die "File '$file' contains unexpected paths: '$unexpected'" + + info "Installing $project release $version from $file" + + sudo tar -C / -xvf "${file}" + + [ -d "$from_dir" ] || die "$project does not exist in expected directory $from_dir" + + for file in "${create_links_for[@]}" + do + local from_path=$(printf "%s/%s" "$from_dir" "$file") + [ -e "$from_path" ] || die "File $from_path not found" + + sudo ln -sf "$from_path" "$link_dir" + done + + info "$project installed\n" +} + +handle_kata() +{ + local version="${1:-}" + + install_kata "$version" + + kata-runtime --version +} + +handle_containerd() +{ + local version="${1:-}" + + install_containerd "$version" + + configure_containerd + + containerd --version +} + +test_installation() +{ + info "Testing $kata_project\n" + + local image="docker.io/library/busybox:latest" + sudo ctr image pull "$image" + + local container_name="test-kata" + + # Used to prove that the kernel in the container + # is different to the host kernel. + local container_kernel=$(sudo ctr run \ + --runtime "$kata_runtime_type" \ + --rm \ + "$image" \ + "$container_name" \ + uname -r || true) + + [ -z "$container_kernel" ] && die "Failed to test $kata_project" + + local host_kernel=$(uname -r) + + info "Test successful:\n" + + info " Host kernel version : $host_kernel" + info " Container kernel version : $container_kernel" + echo +} + +handle_installation() +{ + local kata_version="${1:-}" + local containerd_version="${2:-}" + + setup + + handle_kata "$kata_version" + handle_containerd "$containerd_version" + + test_installation + + info "$kata_project and $containerd_project are now installed" + + echo -e "\n${warnings}\n" +} + +handle_args() +{ + case "${1:-}" in + -h|--help|help) usage; exit 0;; + esac + + local kata_version="${1:-}" + local containerd_version="${2:-}" + + handle_installation \ + "$kata_version" \ + "$containerd_version" +} + +main() +{ + handle_args "$@" +} + +main "$@" From cbdae44992f47300af3dcf3f176a4c4bbde62624 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 9 Oct 2020 17:40:51 -0700 Subject: [PATCH 053/124] agent: fix errorneous parsing for guest block size We were assuming base 10 string before, when the block size from sysfs is actually a hex string. Let's fix that. Fixes: #908 Signed-off-by: Eric Ernst --- src/agent/src/rpc.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index e7933b3ad9..d7b95cd7b6 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -1353,7 +1353,13 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { return Err(anyhow!("Invalid block size")); } - size = v.trim().parse::()?; + size = match u64::from_str_radix(v.trim(), 16) { + Ok(h) => h, + Err(_) => { + warn!(sl!(), "failed to parse the str {} to hex", size); + return Err(anyhow!("Invalid block size")); + } + }; } Err(e) => { info!(sl!(), "memory block size error: {:?}", e.kind()); From 8f0cb2f1ea3a8d97a82e41e383feed37dd4d86bf Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 17:12:26 -0700 Subject: [PATCH 054/124] cgroups: add ability to update CPUSet Add function for applying a cpuset change to a cgroup Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/pkg/cgroups/manager.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/runtime/virtcontainers/pkg/cgroups/manager.go b/src/runtime/virtcontainers/pkg/cgroups/manager.go index fc3462c13e..e0f81777a1 100644 --- a/src/runtime/virtcontainers/pkg/cgroups/manager.go +++ b/src/runtime/virtcontainers/pkg/cgroups/manager.go @@ -331,3 +331,16 @@ func (m *Manager) RemoveDevice(device string) error { m.Unlock() return fmt.Errorf("device %v not found in the cgroup", device) } + +func (m *Manager) SetCPUSet(cpuset string) error { + cgroups, err := m.GetCgroups() + if err != nil { + return err + } + + m.Lock() + cgroups.CpusetCpus = cpuset + m.Unlock() + + return m.Apply() +} From a6d9fd41184b6cc5c0f392c7e139d22c2a334e15 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 17:13:01 -0700 Subject: [PATCH 055/124] sandbox: don't constrain cpus, mem only cpuset, devices Allow for constraining the cpuset as well as the devices-whitelist . Revert sandbox constraints for cpu/memory, as they break the K8S use case. Can re-add behind a non-default flag in the future. The sandbox CPUSet should be updated every time a container is created, updated, or removed. To facilitate this without rewriting the 'non constrained cgroup' handling, let's add to the Sandbox's cgroupsUpdate function. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/container.go | 6 +++ src/runtime/virtcontainers/sandbox.go | 55 ++++++++++++++++++------- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 449b84276e..2c49b14bc5 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -1139,6 +1139,12 @@ func (c *Container) update(resources specs.LinuxResources) error { if q := cpu.Quota; q != nil && *q != 0 { c.config.Resources.CPU.Quota = q } + if cpu.Cpus != "" { + c.config.Resources.CPU.Cpus = cpu.Cpus + } + if cpu.Mems != "" { + c.config.Resources.CPU.Mems = cpu.Mems + } } if c.config.Resources.Memory == nil { diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a393e5c76b..73ea8a95a3 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -564,15 +564,25 @@ func (s *Sandbox) createCgroupManager() error { } spec := s.GetPatchedOCISpec() - if spec != nil { + if spec != nil && spec.Linux != nil { cgroupPath = spec.Linux.CgroupsPath // Kata relies on the cgroup parent created and configured by the container - // engine, but sometimes the sandbox cgroup is not configured and the container - // may have access to all the resources, hence the runtime must constrain the - // sandbox and update the list of devices with the devices hotplugged in the - // hypervisor. - resources = *spec.Linux.Resources + // engine by default. The exception is for devices whitelist as well as sandbox-level + // CPUSet. + if spec.Linux.Resources != nil { + resources.Devices = spec.Linux.Resources.Devices + + if spec.Linux.Resources.CPU != nil { + resources.CPU = &specs.LinuxCPU{ + Cpus: spec.Linux.Resources.CPU.Cpus, + } + } + } + + //TODO: in Docker or Podman use case, it is reasonable to set a constraint. Need to add a flag + // to allow users to configure Kata to constrain CPUs and Memory in this alternative + // scenario. See https://github.com/kata-containers/runtime/issues/2811 } if s.devManager != nil { @@ -1215,7 +1225,7 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro } }() - // Sandbox is reponsable to update VM resources needed by Containers + // Sandbox is responsible to update VM resources needed by Containers // Update resources after having added containers to the sandbox, since // container status is requiered to know if more resources should be added. err = s.updateResources() @@ -1329,6 +1339,11 @@ func (s *Sandbox) DeleteContainer(containerID string) (VCContainer, error) { } } + // update the sandbox cgroup + if err = s.cgroupsUpdate(); err != nil { + return nil, err + } + if err = s.storeSandbox(); err != nil { return nil, err } @@ -1866,11 +1881,12 @@ func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) { return b, nil } -// updateResources will calculate the resources required for the virtual machine, and -// adjust the virtual machine sizing accordingly. For a given sandbox, it will calculate the -// number of vCPUs required based on the sum of container requests, plus default CPUs for the VM. -// Similar is done for memory. If changes in memory or CPU are made, the VM will be updated and -// the agent will online the applicable CPU and memory. +// updateResources will: +// - calculate the resources required for the virtual machine, and adjust the virtual machine +// sizing accordingly. For a given sandbox, it will calculate the number of vCPUs required based +// on the sum of container requests, plus default CPUs for the VM. Similar is done for memory. +// If changes in memory or CPU are made, the VM will be updated and the agent will online the +// applicable CPU and memory. func (s *Sandbox) updateResources() error { if s == nil { return errors.New("sandbox is nil") @@ -1975,9 +1991,20 @@ func (s *Sandbox) GetHypervisorType() string { func (s *Sandbox) cgroupsUpdate() error { // If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already - // in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should - // rely on parent's cgroup CPU/memory values + // in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be + // updated. if s.config.SandboxCgroupOnly { + cpuset, err := s.getSandboxCPUSet() + if err != nil { + return err + } + + if cpuset != "" { + if err := s.cgroupMgr.SetCPUSet(cpuset); err != nil { + return err + } + } + return nil } From 230a9833f8fc094ece903f44d595bbd70ef5075d Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 8 Oct 2020 16:13:14 +1100 Subject: [PATCH 056/124] agent/device: update_spec_device_list() should error if dev not found If update_spec_device_list() is given a device that can't be found in the OCI spec, it currently does nothing, and returns Ok(()). That doesn't seem like what we'd expect and is not what the Go agent in Kata 1 does. Change it to return an error in that case, like Kata 1. Signed-off-by: David Gibson --- src/agent/src/device.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 5dbb166153..8b7e604126 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -262,10 +262,14 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec) -> Result<()> { } } } + return Ok(()); } } - Ok(()) + Err(anyhow!( + "Should have found a matching device {} in the spec", + device.vm_path + )) } // device.Id should be the predicted device name (vda, vdb, ...) From a7ba362f924454ddc53a9ca4edf4c67c6749d58d Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 8 Oct 2020 16:23:36 +1100 Subject: [PATCH 057/124] agent/device: Forward port update_spec_device_list() unit test The Kata 1 Go agent included a unit test for updateSpecDeviceList, but no such unit test exists for the Rust agent's equivalent update_spec_device_list(). Port the Kata1 test to Rust. Signed-off-by: David Gibson --- src/agent/src/device.rs | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 8b7e604126..6e5d8815ba 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -417,4 +417,70 @@ mod tests { assert_eq!(devices[0].major, Some(major)); assert_eq!(devices[0].minor, Some(minor)); } + + #[test] + fn test_update_spec_device_list() { + let (major, minor) = (7, 2); + let mut device = Device::default(); + let mut spec = Spec::default(); + + // container_path empty + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_err()); + + device.container_path = "/dev/null".to_string(); + + // linux is empty + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_err()); + + spec.linux = Some(Linux::default()); + + // linux.devices is empty + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_err()); + + spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice { + path: "/dev/null2".to_string(), + major, + minor, + ..oci::LinuxDevice::default() + }]; + + // vm_path empty + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_err()); + + device.vm_path = "/dev/null".to_string(); + + // guest and host path are not the same + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_err(), "device={:?} spec={:?}", device, spec); + + spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone(); + + // spec.linux.resources is empty + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_ok()); + + // update both devices and cgroup lists + spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice { + path: device.container_path.clone(), + major, + minor, + ..oci::LinuxDevice::default() + }]; + + spec.linux.as_mut().unwrap().resources = Some(oci::LinuxResources { + devices: vec![oci::LinuxDeviceCgroup { + major: Some(major), + minor: Some(minor), + ..oci::LinuxDeviceCgroup::default() + }], + ..oci::LinuxResources::default() + }); + + let res = update_spec_device_list(&device, &mut spec); + assert!(res.is_ok()); + } } From 4978c9092cecf2eb3dad2787ff2bed93d6fb34bf Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 8 Oct 2020 17:01:29 +1100 Subject: [PATCH 058/124] agent/device: Index all devices in spec before updating them The agent needs to update device entries in the OCI spec so that it has the correct major:minor numbers for the guest, which may differ from the host. Entries in the main device list are looked up by device path, but entries in the device resources list are looked up by (host) major:minor. This is done one device at a time, updating as we go in update_spec_device_list(). But since the host and guest have different namespaces, one device might have the same major:minor as a different device on the host. In that case we could update one resource entry to the correct guest values, then mistakenly update it again because it now matches a different host device. To avoid this, rather than looking up and updating one by one, we make all the lookups in advance, creating a map from (host) device path to the indices in the spec where the device and resource entries can be found. Port from the Go agent in Kata 1, https://github.com/kata-containers/agent/commit/d88d46849130 Fixes: #703 Signed-off-by: David Gibson --- src/agent/src/device.rs | 258 ++++++++++++++++++++++++++++++++-------- 1 file changed, 207 insertions(+), 51 deletions(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 6e5d8815ba..89433d34ea 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -28,8 +28,15 @@ macro_rules! sl { const VM_ROOTFS: &str = "/"; +struct DevIndexEntry { + idx: usize, + residx: Vec, +} + +struct DevIndex(HashMap); + // DeviceHandler is the type of callback to be defined to handle every type of device driver. -type DeviceHandler = fn(&Device, &mut Spec, &Arc>) -> Result<()>; +type DeviceHandler = fn(&Device, &mut Spec, &Arc>, &DevIndex) -> Result<()>; // DeviceHandlerList lists the supported drivers. #[cfg_attr(rustfmt, rustfmt_skip)] @@ -194,7 +201,7 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { // the same device in the list of devices provided through the OCI spec. // This is needed to update information about minor/major numbers that cannot // be predicted from the caller. -fn update_spec_device_list(device: &Device, spec: &mut Spec) -> Result<()> { +fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) -> Result<()> { let major_id: c_uint; let minor_id: c_uint; @@ -228,48 +235,44 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec) -> Result<()> { "got the device: dev_path: {}, major: {}, minor: {}\n", &device.vm_path, major_id, minor_id ); - let devices = linux.devices.as_mut_slice(); - for dev in devices.iter_mut() { - if dev.path == device.container_path { - let host_major = dev.major; - let host_minor = dev.minor; + if let Some(idxdata) = devidx.0.get(device.container_path.as_str()) { + let dev = &mut linux.devices[idxdata.idx]; + let host_major = dev.major; + let host_minor = dev.minor; - dev.major = major_id as i64; - dev.minor = minor_id as i64; + dev.major = major_id as i64; + dev.minor = minor_id as i64; + + info!( + sl!(), + "change the device from major: {} minor: {} to vm device major: {} minor: {}", + host_major, + host_minor, + major_id, + minor_id + ); + + // Resources must be updated since they are used to identify + // the device in the devices cgroup. + for ridx in &idxdata.residx { + // unwrap is safe, because residx would be empty if there + // were no resources + let res = &mut linux.resources.as_mut().unwrap().devices[*ridx]; + res.major = Some(major_id as i64); + res.minor = Some(minor_id as i64); info!( sl!(), - "change the device from major: {} minor: {} to vm device major: {} minor: {}", - host_major, - host_minor, - major_id, - minor_id + "set resources for device major: {} minor: {}\n", major_id, minor_id ); - - // Resources must be updated since they are used to identify the - // device in the devices cgroup. - if let Some(res) = linux.resources.as_mut() { - let ds = res.devices.as_mut_slice(); - for d in ds.iter_mut() { - if d.major == Some(host_major) && d.minor == Some(host_minor) { - d.major = Some(major_id as i64); - d.minor = Some(minor_id as i64); - - info!( - sl!(), - "set resources for device major: {} minor: {}\n", major_id, minor_id - ); - } - } - } - return Ok(()); } + Ok(()) + } else { + Err(anyhow!( + "Should have found a matching device {} in the spec", + device.vm_path + )) } - - Err(anyhow!( - "Should have found a matching device {} in the spec", - device.vm_path - )) } // device.Id should be the predicted device name (vda, vdb, ...) @@ -278,12 +281,13 @@ fn virtiommio_blk_device_handler( device: &Device, spec: &mut Spec, _sandbox: &Arc>, + devidx: &DevIndex, ) -> Result<()> { if device.vm_path == "" { return Err(anyhow!("Invalid path for virtio mmio blk device")); } - update_spec_device_list(device, spec) + update_spec_device_list(device, spec, devidx) } // device.Id should be the PCI address in the format "bridgeAddr/deviceAddr". @@ -293,6 +297,7 @@ fn virtio_blk_device_handler( device: &Device, spec: &mut Spec, sandbox: &Arc>, + devidx: &DevIndex, ) -> Result<()> { let mut dev = device.clone(); @@ -302,7 +307,7 @@ fn virtio_blk_device_handler( dev.vm_path = get_pci_device_name(sandbox, &device.id)?; } - update_spec_device_list(&dev, spec) + update_spec_device_list(&dev, spec, devidx) } // device.Id should be the SCSI address of the disk in the format "scsiID:lunID" @@ -310,22 +315,46 @@ fn virtio_scsi_device_handler( device: &Device, spec: &mut Spec, sandbox: &Arc>, + devidx: &DevIndex, ) -> Result<()> { let mut dev = device.clone(); dev.vm_path = get_scsi_device_name(sandbox, &device.id)?; - update_spec_device_list(&dev, spec) + update_spec_device_list(&dev, spec, devidx) } fn virtio_nvdimm_device_handler( device: &Device, spec: &mut Spec, _sandbox: &Arc>, + devidx: &DevIndex, ) -> Result<()> { if device.vm_path == "" { return Err(anyhow!("Invalid path for nvdimm device")); } - update_spec_device_list(device, spec) + update_spec_device_list(device, spec, devidx) +} + +impl DevIndex { + fn new(spec: &Spec) -> DevIndex { + let mut map = HashMap::new(); + + for linux in spec.linux.as_ref() { + for (i, d) in linux.devices.iter().enumerate() { + let mut residx = Vec::new(); + + for linuxres in linux.resources.as_ref() { + for (j, r) in linuxres.devices.iter().enumerate() { + if r.major == Some(d.major) && r.minor == Some(d.minor) { + residx.push(j); + } + } + } + map.insert(d.path.clone(), DevIndexEntry { idx: i, residx }); + } + } + DevIndex(map) + } } pub fn add_devices( @@ -333,14 +362,21 @@ pub fn add_devices( spec: &mut Spec, sandbox: &Arc>, ) -> Result<()> { + let devidx = DevIndex::new(spec); + for device in devices.iter() { - add_device(device, spec, sandbox)?; + add_device(device, spec, sandbox, &devidx)?; } Ok(()) } -fn add_device(device: &Device, spec: &mut Spec, sandbox: &Arc>) -> Result<()> { +fn add_device( + device: &Device, + spec: &mut Spec, + sandbox: &Arc>, + devidx: &DevIndex, +) -> Result<()> { // log before validation to help with debugging gRPC protocol version differences. info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}", device.id, device.field_type, device.vm_path, device.container_path, device.options); @@ -359,7 +395,7 @@ fn add_device(device: &Device, spec: &mut Spec, sandbox: &Arc>) - match DEVICEHANDLERLIST.get(device.field_type.as_str()) { None => Err(anyhow!("Unknown device type {}", device.field_type)), - Some(dev_handler) => dev_handler(device, spec, sandbox), + Some(dev_handler) => dev_handler(device, spec, sandbox, devidx), } } @@ -425,19 +461,22 @@ mod tests { let mut spec = Spec::default(); // container_path empty - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_err()); device.container_path = "/dev/null".to_string(); // linux is empty - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_err()); spec.linux = Some(Linux::default()); // linux.devices is empty - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_err()); spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice { @@ -448,19 +487,22 @@ mod tests { }]; // vm_path empty - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_err()); device.vm_path = "/dev/null".to_string(); // guest and host path are not the same - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_err(), "device={:?} spec={:?}", device, spec); spec.linux.as_mut().unwrap().devices[0].path = device.container_path.clone(); // spec.linux.resources is empty - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_ok()); // update both devices and cgroup lists @@ -480,7 +522,121 @@ mod tests { ..oci::LinuxResources::default() }); - let res = update_spec_device_list(&device, &mut spec); + let devidx = DevIndex::new(&spec); + let res = update_spec_device_list(&device, &mut spec, &devidx); assert!(res.is_ok()); } + + #[test] + fn test_update_spec_device_list_guest_host_conflict() { + let null_rdev = fs::metadata("/dev/null").unwrap().rdev(); + let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev(); + let full_rdev = fs::metadata("/dev/full").unwrap().rdev(); + + let host_major_a = stat::major(null_rdev) as i64; + let host_minor_a = stat::minor(null_rdev) as i64; + let host_major_b = stat::major(zero_rdev) as i64; + let host_minor_b = stat::minor(zero_rdev) as i64; + + let mut spec = Spec { + linux: Some(Linux { + devices: vec![ + oci::LinuxDevice { + path: "/dev/a".to_string(), + r#type: "c".to_string(), + major: host_major_a, + minor: host_minor_a, + ..oci::LinuxDevice::default() + }, + oci::LinuxDevice { + path: "/dev/b".to_string(), + r#type: "c".to_string(), + major: host_major_b, + minor: host_minor_b, + ..oci::LinuxDevice::default() + }, + ], + resources: Some(LinuxResources { + devices: vec![ + oci::LinuxDeviceCgroup { + r#type: "c".to_string(), + major: Some(host_major_a), + minor: Some(host_minor_a), + ..oci::LinuxDeviceCgroup::default() + }, + oci::LinuxDeviceCgroup { + r#type: "c".to_string(), + major: Some(host_major_b), + minor: Some(host_minor_b), + ..oci::LinuxDeviceCgroup::default() + }, + ], + ..LinuxResources::default() + }), + ..Linux::default() + }), + ..Spec::default() + }; + let devidx = DevIndex::new(&spec); + + let dev_a = Device { + container_path: "/dev/a".to_string(), + vm_path: "/dev/zero".to_string(), + ..Device::default() + }; + + let guest_major_a = stat::major(zero_rdev) as i64; + let guest_minor_a = stat::minor(zero_rdev) as i64; + + let dev_b = Device { + container_path: "/dev/b".to_string(), + vm_path: "/dev/full".to_string(), + ..Device::default() + }; + + let guest_major_b = stat::major(full_rdev) as i64; + let guest_minor_b = stat::minor(full_rdev) as i64; + + let specdevices = &spec.linux.as_ref().unwrap().devices; + assert_eq!(host_major_a, specdevices[0].major); + assert_eq!(host_minor_a, specdevices[0].minor); + assert_eq!(host_major_b, specdevices[1].major); + assert_eq!(host_minor_b, specdevices[1].minor); + + let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); + assert_eq!(Some(host_major_a), specresources.devices[0].major); + assert_eq!(Some(host_minor_a), specresources.devices[0].minor); + assert_eq!(Some(host_major_b), specresources.devices[1].major); + assert_eq!(Some(host_minor_b), specresources.devices[1].minor); + + let res = update_spec_device_list(&dev_a, &mut spec, &devidx); + assert!(res.is_ok()); + + let specdevices = &spec.linux.as_ref().unwrap().devices; + assert_eq!(guest_major_a, specdevices[0].major); + assert_eq!(guest_minor_a, specdevices[0].minor); + assert_eq!(host_major_b, specdevices[1].major); + assert_eq!(host_minor_b, specdevices[1].minor); + + let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); + assert_eq!(Some(guest_major_a), specresources.devices[0].major); + assert_eq!(Some(guest_minor_a), specresources.devices[0].minor); + assert_eq!(Some(host_major_b), specresources.devices[1].major); + assert_eq!(Some(host_minor_b), specresources.devices[1].minor); + + let res = update_spec_device_list(&dev_b, &mut spec, &devidx); + assert!(res.is_ok()); + + let specdevices = &spec.linux.as_ref().unwrap().devices; + assert_eq!(guest_major_a, specdevices[0].major); + assert_eq!(guest_minor_a, specdevices[0].minor); + assert_eq!(guest_major_b, specdevices[1].major); + assert_eq!(guest_minor_b, specdevices[1].minor); + + let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); + assert_eq!(Some(guest_major_a), specresources.devices[0].major); + assert_eq!(Some(guest_minor_a), specresources.devices[0].minor); + assert_eq!(Some(guest_major_b), specresources.devices[1].major); + assert_eq!(Some(guest_minor_b), specresources.devices[1].minor); + } } From 9c16643c1231219e918f8e9bfb0952c600421b64 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 8 Oct 2020 17:15:14 +1100 Subject: [PATCH 059/124] agent/device: Check type as well as major:minor when looking up devices To update device resource entries from host to guest, we search for the right entry by host major:minor numbers, then later update it. However block and character devices exist in separate major:minor namespaces so we could have one block and one character device with matching major:minor and thus incorrectly update both with the details for whichever device is processed second. Add a check on device type to prevent this. Port from the Kata 1 Go agent https://github.com/kata-containers/agent/commit/27ebdc9d2761 Fixes: #703 Signed-off-by: David Gibson --- src/agent/src/device.rs | 78 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 89433d34ea..18f53e0c9e 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -345,7 +345,10 @@ impl DevIndex { for linuxres in linux.resources.as_ref() { for (j, r) in linuxres.devices.iter().enumerate() { - if r.major == Some(d.major) && r.minor == Some(d.minor) { + if r.r#type == d.r#type + && r.major == Some(d.major) + && r.minor == Some(d.minor) + { residx.push(j); } } @@ -639,4 +642,77 @@ mod tests { assert_eq!(Some(guest_major_b), specresources.devices[1].major); assert_eq!(Some(guest_minor_b), specresources.devices[1].minor); } + + #[test] + fn test_update_spec_device_list_char_block_conflict() { + let null_rdev = fs::metadata("/dev/null").unwrap().rdev(); + + let guest_major = stat::major(null_rdev) as i64; + let guest_minor = stat::minor(null_rdev) as i64; + let host_major: i64 = 99; + let host_minor: i64 = 99; + + let mut spec = Spec { + linux: Some(Linux { + devices: vec![ + oci::LinuxDevice { + path: "/dev/char".to_string(), + r#type: "c".to_string(), + major: host_major, + minor: host_minor, + ..oci::LinuxDevice::default() + }, + oci::LinuxDevice { + path: "/dev/block".to_string(), + r#type: "b".to_string(), + major: host_major, + minor: host_minor, + ..oci::LinuxDevice::default() + }, + ], + resources: Some(LinuxResources { + devices: vec![ + LinuxDeviceCgroup { + r#type: "c".to_string(), + major: Some(host_major), + minor: Some(host_minor), + ..LinuxDeviceCgroup::default() + }, + LinuxDeviceCgroup { + r#type: "b".to_string(), + major: Some(host_major), + minor: Some(host_minor), + ..LinuxDeviceCgroup::default() + }, + ], + ..LinuxResources::default() + }), + ..Linux::default() + }), + ..Spec::default() + }; + let devidx = DevIndex::new(&spec); + + let dev = Device { + container_path: "/dev/char".to_string(), + vm_path: "/dev/null".to_string(), + ..Device::default() + }; + + let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); + assert_eq!(Some(host_major), specresources.devices[0].major); + assert_eq!(Some(host_minor), specresources.devices[0].minor); + assert_eq!(Some(host_major), specresources.devices[1].major); + assert_eq!(Some(host_minor), specresources.devices[1].minor); + + let res = update_spec_device_list(&dev, &mut spec, &devidx); + assert!(res.is_ok()); + + // Only the char device, not the block device should be updated + let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap(); + assert_eq!(Some(guest_major), specresources.devices[0].major); + assert_eq!(Some(guest_minor), specresources.devices[0].minor); + assert_eq!(Some(host_major), specresources.devices[1].major); + assert_eq!(Some(host_minor), specresources.devices[1].minor); + } } From e884fef48330e858f49268c7320986c2bc12ff37 Mon Sep 17 00:00:00 2001 From: Ychau Wang Date: Tue, 13 Oct 2020 11:13:17 +0800 Subject: [PATCH 060/124] docs: update the build kata containers kernel document Update the build kata containers kernel document for 2.0 release. Fixed the 1.x release project paths and urls, using the kata-containers project file paths and urls. Fixes: #929 Signed-off-by: Ychau Wang --- tools/packaging/kernel/README.md | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index 05d5afa38a..7e13188ecf 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -63,25 +63,25 @@ $ ./build-kernel.sh -v 4.19.86 -g nvidia -f -d setup > **Note** > - `-v 4.19.86`: Specify the guest kernel version. > - `-g nvidia`: To build a guest kernel supporting Nvidia GPU. -> - `-f`: The .config file is forced to be generated even if the kernel directory already exists. +> - `-f`: The `.config` file is forced to be generated even if the kernel directory already exists. > - `-d`: Enable bash debug mode. ## Setup kernel source code ```bash -$ go get -d -u github.com/kata-containers/packaging -$ cd $GOPATH/src/github.com/kata-containers/packaging/kernel +$ go get -d -u github.com/kata-containers/kata-containers +$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kernel $ ./build-kernel.sh setup ``` The script `./build-kernel.sh` tries to apply the patches from -`${GOPATH}/src/github.com/kata-containers/packaging/kernel/patches/` when it +`${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/patches/` when it sets up a kernel. If you want to add a source modification, add a patch on this directory. The script also adds a kernel config file from -`${GOPATH}/src/github.com/kata-containers/packaging/kernel/configs/` to `.config` +`${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/configs/` to `.config` in the kernel source code. You can modify it as needed. ## Build the kernel @@ -106,7 +106,7 @@ $ ./build-kernel.sh install Kata Containers packaging repository holds the kernel configs and patches. The config and patches can work for many versions, but we only test the -kernel version defined in the [runtime versions file][runtime-versions-file]. +kernel version defined in the [Kata Containers versions file][kata-containers-versions-file]. For further details, see [the kernel configuration documentation](configs). @@ -115,33 +115,33 @@ For further details, see [the kernel configuration documentation](configs). The Kata Containers CI scripts install the kernel from [CI cache job][cache-job] or build from sources. -If the kernel defined in the [runtime versions file][runtime-versions-file] is +If the kernel defined in the [Kata Containers versions file][kata-containers-versions-file] is built and cached with the latest kernel config and patches, it installs. Otherwise, the kernel is built from source. -The Kata kernel version is a mix of the kernel version defined in the [runtime -versions file][runtime-versions-file] and the file `kata_config_version`. This +The Kata kernel version is a mix of the kernel version defined in the [Kata Containers +versions file][kata-containers-versions-file] and the file `kata_config_version`. This helps to identify if a kernel build has the latest recommend configuration. Example: ```bash -# From https://github.com/kata-containers/runtime/blob/master/versions.yaml -$ kernel_version_in_versions_file=4.10.1 -# From https://github.com/kata-containers/packaging/blob/master/kernel/kata_config_version -$ kata_config_version=25 +# From https://github.com/kata-containers/kata-containers/blob/2.0-dev/versions.yaml +$ kernel_version_in_versions_file=5.4.60 +# From https://github.com/kata-containers/kata-containers/blob/2.0-dev/tools/packaging/kernel/kata_config_version +$ kata_config_version=83 $ latest_kernel_version=${kernel_version_in_versions_file}-${kata_config_version} ``` -The resulting version is 4.10.1-25, this helps identify whether or not the kernel +The resulting version is 5.4.60-83, this helps identify whether or not the kernel configs are up-to-date on a CI version. ## Contribute In order to do Kata Kernel changes. There are places to contribute: -1. [Kata runtime versions file][runtime-versions-file]: This file points to the +1. [Kata Containers versions file][kata-containers-versions-file]: This file points to the recommended versions to be used by Kata. To update the kernel version send a pull request to update that version. The Kata CI will run all the use cases and verify it works. @@ -174,7 +174,7 @@ In this case, the PR you submit needs to be tested together with a patch from another Kata Containers repository. To do this you have to specify which repository and which pull request [it depends on][depends-on-docs]. -[runtime-versions-file]: https://github.com/kata-containers/runtime/blob/master/versions.yaml -[patches-dir]: https://github.com/kata-containers/packaging/tree/master/kernel/patches +[kata-containers-versions-file]: ../../../versions.yaml +[patches-dir]: patches [depends-on-docs]: https://github.com/kata-containers/tests/blob/master/README.md#breaking-compatibility [cache-job]: http://jenkins.katacontainers.io/job/image-nightly-x86_64/ From ce54090f25ff1d82e66f0973eb3a34bd84bfb1d4 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 13 Oct 2020 11:20:44 +0100 Subject: [PATCH 061/124] docs: Update upgrading guide Update the upgrading guide for 2.0. Fixes: #928. Signed-off-by: James O. D. Hunt --- docs/Upgrading.md | 249 +++++++++++++++++++--------------------------- 1 file changed, 102 insertions(+), 147 deletions(-) diff --git a/docs/Upgrading.md b/docs/Upgrading.md index cd30b0ae0e..3d8a554927 100644 --- a/docs/Upgrading.md +++ b/docs/Upgrading.md @@ -1,185 +1,140 @@ * [Introduction](#introduction) -* [Unsupported scenarios](#unsupported-scenarios) -* [Maintenance Warning](#maintenance-warning) -* [Upgrade from Clear Containers](#upgrade-from-clear-containers) - * [Stop all running Clear Container instances](#stop-all-running-clear-container-instances) - * [Configuration migration](#configuration-migration) - * [Remove Clear Containers packages](#remove-clear-containers-packages) - * [Fedora](#fedora) - * [Ubuntu](#ubuntu) - * [Disable old container manager configuration](#disable-old-container-manager-configuration) - * [Install Kata Containers](#install-kata-containers) - * [Create a Kata Container](#create-a-kata-container) -* [Upgrade from runV](#upgrade-from-runv) +* [Maintenance warning](#maintenance-warning) +* [Determine current version](#determine-current-version) +* [Determine latest version](#determine-latest-version) +* [Configuration changes](#configuration-changes) * [Upgrade Kata Containers](#upgrade-kata-containers) -* [Appendices](#appendices) - * [Assets](#assets) - * [Guest kernel](#guest-kernel) - * [Image](#image) - * [Determining asset versions](#determining-asset-versions) + * [Upgrade native distribution packaged version](#upgrade-native-distribution-packaged-version) + * [Static installation](#static-installation) + * [Determine if you are using a static installation](#determine-if-you-are-using-a-static-installation) + * [Remove a static installation](#remove-a-static-installation) + * [Upgrade a static installation](#upgrade-a-static-installation) +* [Custom assets](#custom-assets) # Introduction -This document explains how to upgrade from -[Clear Containers](https://github.com/clearcontainers) and [runV](https://github.com/hyperhq/runv) to -[Kata Containers](https://github.com/kata-containers) and how to upgrade an existing -Kata Containers system to the latest version. +This document outlines the options for upgrading from a +[Kata Containers 1.x release](https://github.com/kata-containers/runtime/releases) to a +[Kata Containers 2.x release](https://github.com/kata-containers/kata-containers/releases). -# Unsupported scenarios +# Maintenance warning -Upgrading a Clear Containers system on the following distributions is **not** -supported since the installation process for these distributions makes use of -unpackaged components: +Kata Containers 2.x is the new focus for the Kata Containers development +community. -- [CentOS](https://github.com/clearcontainers/runtime/blob/master/docs/centos-installation-guide.md) -- [BCLinux](https://github.com/clearcontainers/runtime/blob/master/docs/bclinux-installation-guide.md) -- [RHEL](https://github.com/clearcontainers/runtime/blob/master/docs/rhel-installation-guide.md) -- [SLES](https://github.com/clearcontainers/runtime/blob/master/docs/sles-installation-guide.md) +Although Kata Containers 1.x releases will continue to be published for a +period of time, once a stable release for Kata Containers 2.x is published, +Kata Containers 1.x stable users should consider switching to the Kata 2.x +release. -Additionally, upgrading -[Clear Linux](https://github.com/clearcontainers/runtime/blob/master/docs/clearlinux-installation-guide.md) -is not supported as Kata Containers packages do not yet exist. +See the [stable branch strategy documentation](Stable-Branch-Strategy.md) for +further details. -# Maintenance Warning +# Determine current version -The Clear Containers codebase is no longer being developed. Only new releases -will be considered for significant bug fixes. +To display the current Kata Containers version, run one of the following: -The main development focus is now on Kata Containers. All Clear Containers -users are encouraged to switch to Kata Containers. - -# Upgrade from Clear Containers - -Since Kata Containers can co-exist on the same system as Clear Containers, if -you already have Clear Containers installed, the upgrade process is simply to -install Kata Containers. However, since Clear Containers is -[no longer being actively developed](#maintenance-warning), -you are encouraged to remove Clear Containers from your systems. - -## Stop all running Clear Container instances - -Assuming a Docker\* system, to stop all currently running Clear Containers: - -``` -$ for container in $(sudo docker ps -q); do sudo docker stop $container; done +```bash +$ kata-runtime --version +$ containerd-shim-kata-v2 --version ``` -## Configuration migration +# Determine latest version -The automatic migration of -[Clear Containers configuration](https://github.com/clearcontainers/runtime#configuration) to -[Kata Containers configuration](../src/runtime/README.md#configuration) is -not supported. +Kata Containers 2.x releases are published on the +[Kata Containers GitHub releases page](https://github.com/kata-containers/kata-containers/releases). -If you have made changes to your Clear Containers configuration, you should -review those changes and decide whether to manually apply those changes to the -Kata Containers configuration. +Alternatively, if you are using Kata Containers version 1.12.0 or newer, you +can check for newer releases using the command line: -> **Note**: This step must be completed before continuing to -> [remove the Clear Containers packages](#remove-clear-containers-packages) since doing so will -> *delete the default Clear Containers configuration file from your system*. - -## Remove Clear Containers packages - -> **Warning**: If you have modified your -> [Clear Containers configuration](https://github.com/clearcontainers/runtime#configuration), -> you might want to make a safe copy of the configuration file before removing the -> packages since doing so will *delete the default configuration file* - -### Fedora - -``` -$ sudo -E dnf remove cc-runtime\* cc-proxy\* cc-shim\* linux-container clear-containers-image qemu-lite cc-ksm-throttler -$ sudo rm /etc/yum.repos.d/home:clearcontainers:clear-containers-3.repo +```bash +$ kata-runtime kata-check --check-version-only ``` -### Ubuntu +There are various other related options. Run `kata-runtime kata-check --help` +for further details. -``` -$ sudo apt-get purge cc-runtime\* cc-proxy\* cc-shim\* linux-container clear-containers-image qemu-lite cc-ksm-throttler -$ sudo rm /etc/apt/sources.list.d/clear-containers.list -``` +# Configuration changes -## Disable old container manager configuration +The [Kata Containers 2.x configuration file](/src/runtime/README.md#configuration) +is compatible with the +[Kata Containers 1.x configuration file](https://github.com/kata-containers/runtime/blob/master/README.md#configuration). -Assuming a Docker installation, remove the docker configuration for Clear -Containers: +However, if you have created a local configuration file +(`/etc/kata-containers/configuration.toml`), this will mask the newer Kata +Containers 2.x configuration file. -``` -$ sudo rm /etc/systemd/system/docker.service.d/clear-containers.conf -``` - -## Install Kata Containers - -Follow one of the [installation guides](install). - -## Create a Kata Container - -``` -$ sudo docker run -ti busybox sh -``` - -# Upgrade from runV - -runV and Kata Containers can run together on the same system without affecting each other, as long as they are -not configured to use the same container root storage. Currently, runV defaults to `/run/runv` and Kata Containers -defaults to `/var/run/kata-containers`. - -Now, to upgrade from runV you need to fresh install Kata Containers by following one of -the [installation guides](install). +Since Kata Containers 2.x introduces a number of new options and changes +some default values, we recommend that you disable the local configuration +file (by moving or renaming it) until you have reviewed the changes to the +official configuration file and applied them to your local file if required. # Upgrade Kata Containers +## Upgrade native distribution packaged version + As shown in the [installation instructions](install), Kata Containers provide binaries for popular distributions in their native packaging formats. This allows Kata Containers to be upgraded using the standard package management tools for your distribution. -# Appendices +> **Note:** +> +> Users should prefer the distribution packaged version of Kata Containers +> unless they understand the implications of a manual installation. -## Assets +## Static installation -Kata Containers requires additional resources to create a virtual machine -container. These resources are called -[Kata Containers assets](./design/architecture.md#assets), -which comprise a guest kernel and a root filesystem or initrd image. This -section describes when these components are updated. +> **Note:** +> +> Unless you are an advanced user, if you are using a static installation of +> Kata Containers, we recommend you remove it and install a +> [native distribution packaged version](#upgrade-native-distribution-packaged-version) +> instead. -Since the official assets are packaged, they are automatically upgraded when -new package versions are published. +### Determine if you are using a static installation -> **Warning**: Note that if you use custom assets (by modifying the -> [Kata Runtime configuration > file](../src/runtime/README.md#configuration)), -> it is your responsibility to ensure they are updated as necessary. - -### Guest kernel - -The `kata-linux-container` package contains a Linux\* kernel based on the -latest vanilla version of the -[long-term kernel](https://www.kernel.org/) -plus a small number of -[patches](../tools/packaging/kernel). - -The `Longterm` branch is only updated with -[important bug fixes](https://www.kernel.org/category/releases.html) -meaning this package is only updated when necessary. - -The guest kernel package is updated when a new long-term kernel is released -and when any patch updates are required. - -### Image - -The `kata-containers-image` package is updated only when critical updates are -available for the packages used to create it, such as: - -- systemd -- [Kata Containers Agent](../src/agent) - -### Determining asset versions - -To see which versions of the assets being used: +If the following command displays the output "static", you are using a static +version of Kata Containers: +```bash +$ ls /opt/kata/bin/kata-runtime &>/dev/null && echo static ``` -$ kata-runtime kata-env -``` + +### Remove a static installation + +Static installations are installed in `/opt/kata/`, so to uninstall simply +remove this directory. + +### Upgrade a static installation + +If you understand the implications of using a static installation, to upgrade +first +[remove the existing static installation](#remove-a-static-installation), then +[install the latest release](#determine-latest-version). + +See the +[manual installation installation documentation](install/README.md#manual-installation) +for details on how to automatically install and configuration a static release +with containerd. + +# Custom assets + +> **Note:** +> +> This section only applies to advanced users who have built their own guest +> kernel or image. + +If you are using custom +[guest assets](design/architecture.md#guest-assets), +you must upgrade them to work with Kata Containers 2.x since Kata +Containers 1.x assets will **not** work. + +See the following for further details: + +- [Guest kernel documentation](/tools/packaging/kernel) +- [Guest image and initrd documentation](/tools/osbuilder) + +The official assets are packaged meaning they are automatically included in +new releases. From a441f21c40ad528a22950d9a0aefe3e08a5c863e Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 18:10:27 -0700 Subject: [PATCH 062/124] cpuset: add cpuset pkg Pulled from 1.18.4 Kubernetes, adding the cpuset pkg for managing CPUSet calculations on the host. Go mod'ing the original code from k8s.io/kubernetes was very painful, and this is very static, so let's just pull in what we need. Signed-off-by: Eric Ernst --- src/runtime/go.sum | 1 + .../virtcontainers/pkg/cpuset/cpuset.go | 296 +++++++++++++++ .../virtcontainers/pkg/cpuset/cpuset_test.go | 348 ++++++++++++++++++ src/runtime/virtcontainers/sandbox.go | 1 + 4 files changed, 646 insertions(+) create mode 100644 src/runtime/virtcontainers/pkg/cpuset/cpuset.go create mode 100644 src/runtime/virtcontainers/pkg/cpuset/cpuset_test.go diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 80e20e35fc..5ad94ef100 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -179,6 +179,7 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/src/runtime/virtcontainers/pkg/cpuset/cpuset.go b/src/runtime/virtcontainers/pkg/cpuset/cpuset.go new file mode 100644 index 0000000000..ee944c9714 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cpuset/cpuset.go @@ -0,0 +1,296 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Copyright (c) 2017 The Kubernetes Authors +// SPDX-License-Identifier: Apache-2.0 + +package cpuset + +import ( + "bytes" + "fmt" + "reflect" + "sort" + "strconv" + "strings" +) + +// Builder is a mutable builder for CPUSet. Functions that mutate instances +// of this type are not thread-safe. +type Builder struct { + result CPUSet + done bool +} + +// NewBuilder returns a mutable CPUSet builder. +func NewBuilder() Builder { + return Builder{ + result: CPUSet{ + elems: map[int]struct{}{}, + }, + } +} + +// Add adds the supplied elements to the result. Calling Add after calling +// Result has no effect. +func (b Builder) Add(elems ...int) { + if b.done { + return + } + for _, elem := range elems { + b.result.elems[elem] = struct{}{} + } +} + +// Result returns the result CPUSet containing all elements that were +// previously added to this builder. Subsequent calls to Add have no effect. +func (b Builder) Result() CPUSet { + b.done = true + return b.result +} + +// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs. +type CPUSet struct { + elems map[int]struct{} +} + +// NewCPUSet returns a new CPUSet containing the supplied elements. +func NewCPUSet(cpus ...int) CPUSet { + b := NewBuilder() + for _, c := range cpus { + b.Add(c) + } + return b.Result() +} + +// Size returns the number of elements in this set. +func (s CPUSet) Size() int { + return len(s.elems) +} + +// IsEmpty returns true if there are zero elements in this set. +func (s CPUSet) IsEmpty() bool { + return s.Size() == 0 +} + +// Contains returns true if the supplied element is present in this set. +func (s CPUSet) Contains(cpu int) bool { + _, found := s.elems[cpu] + return found +} + +// Equals returns true if the supplied set contains exactly the same elements +// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s). +func (s CPUSet) Equals(s2 CPUSet) bool { + return reflect.DeepEqual(s.elems, s2.elems) +} + +// Filter returns a new CPU set that contains all of the elements from this +// set that match the supplied predicate, without mutating the source set. +func (s CPUSet) Filter(predicate func(int) bool) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + if predicate(cpu) { + b.Add(cpu) + } + } + return b.Result() +} + +// FilterNot returns a new CPU set that contains all of the elements from this +// set that do not match the supplied predicate, without mutating the source +// set. +func (s CPUSet) FilterNot(predicate func(int) bool) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + if !predicate(cpu) { + b.Add(cpu) + } + } + return b.Result() +} + +// IsSubsetOf returns true if the supplied set contains all the elements +func (s CPUSet) IsSubsetOf(s2 CPUSet) bool { + result := true + for cpu := range s.elems { + if !s2.Contains(cpu) { + result = false + break + } + } + return result +} + +// Union returns a new CPU set that contains all of the elements from this +// set and all of the elements from the supplied set, without mutating +// either source set. +func (s CPUSet) Union(s2 CPUSet) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + b.Add(cpu) + } + for cpu := range s2.elems { + b.Add(cpu) + } + return b.Result() +} + +// UnionAll returns a new CPU set that contains all of the elements from this +// set and all of the elements from the supplied sets, without mutating +// either source set. +func (s CPUSet) UnionAll(s2 []CPUSet) CPUSet { + b := NewBuilder() + for cpu := range s.elems { + b.Add(cpu) + } + for _, cs := range s2 { + for cpu := range cs.elems { + b.Add(cpu) + } + } + return b.Result() +} + +// Intersection returns a new CPU set that contains all of the elements +// that are present in both this set and the supplied set, without mutating +// either source set. +func (s CPUSet) Intersection(s2 CPUSet) CPUSet { + return s.Filter(func(cpu int) bool { return s2.Contains(cpu) }) +} + +// Difference returns a new CPU set that contains all of the elements that +// are present in this set and not the supplied set, without mutating either +// source set. +func (s CPUSet) Difference(s2 CPUSet) CPUSet { + return s.FilterNot(func(cpu int) bool { return s2.Contains(cpu) }) +} + +// ToSlice returns a slice of integers that contains all elements from +// this set. +func (s CPUSet) ToSlice() []int { + result := []int{} + for cpu := range s.elems { + result = append(result, cpu) + } + sort.Ints(result) + return result +} + +// ToSliceNoSort returns a slice of integers that contains all elements from +// this set. +func (s CPUSet) ToSliceNoSort() []int { + result := []int{} + for cpu := range s.elems { + result = append(result, cpu) + } + return result +} + +// String returns a new string representation of the elements in this CPU set +// in canonical linux CPU list format. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func (s CPUSet) String() string { + if s.IsEmpty() { + return "" + } + + elems := s.ToSlice() + + type rng struct { + start int + end int + } + + ranges := []rng{{elems[0], elems[0]}} + + for i := 1; i < len(elems); i++ { + lastRange := &ranges[len(ranges)-1] + // if this element is adjacent to the high end of the last range + if elems[i] == lastRange.end+1 { + // then extend the last range to include this element + lastRange.end = elems[i] + continue + } + // otherwise, start a new range beginning with this element + ranges = append(ranges, rng{elems[i], elems[i]}) + } + + // construct string from ranges + var result bytes.Buffer + for _, r := range ranges { + if r.start == r.end { + result.WriteString(strconv.Itoa(r.start)) + } else { + result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end)) + } + result.WriteString(",") + } + return strings.TrimRight(result.String(), ",") +} + +// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string. +// +// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS +func Parse(s string) (CPUSet, error) { + b := NewBuilder() + + // Handle empty string. + if s == "" { + return b.Result(), nil + } + + // Split CPU list string: + // "0-5,34,46-48 => ["0-5", "34", "46-48"] + ranges := strings.Split(s, ",") + + for _, r := range ranges { + boundaries := strings.Split(r, "-") + if len(boundaries) == 1 { + // Handle ranges that consist of only one element like "34". + elem, err := strconv.Atoi(boundaries[0]) + if err != nil { + return NewCPUSet(), err + } + b.Add(elem) + } else if len(boundaries) == 2 { + // Handle multi-element ranges like "0-5". + start, err := strconv.Atoi(boundaries[0]) + if err != nil { + return NewCPUSet(), err + } + end, err := strconv.Atoi(boundaries[1]) + if err != nil { + return NewCPUSet(), err + } + // Add all elements to the result. + // e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48]. + for e := start; e <= end; e++ { + b.Add(e) + } + } + } + return b.Result(), nil +} + +// Clone returns a copy of this CPU set. +func (s CPUSet) Clone() CPUSet { + b := NewBuilder() + for elem := range s.elems { + b.Add(elem) + } + return b.Result() +} diff --git a/src/runtime/virtcontainers/pkg/cpuset/cpuset_test.go b/src/runtime/virtcontainers/pkg/cpuset/cpuset_test.go new file mode 100644 index 0000000000..433a702d22 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cpuset/cpuset_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Copyright (c) 2017 The Kubernetes Authors +// SPDX-License-Identifier: Apache-2.0 + +package cpuset + +import ( + "reflect" + "testing" +) + +func TestCPUSetBuilder(t *testing.T) { + b := NewBuilder() + elems := []int{1, 2, 3, 4, 5} + for _, elem := range elems { + b.Add(elem) + } + result := b.Result() + for _, elem := range elems { + if !result.Contains(elem) { + t.Fatalf("expected cpuset to contain element %d: [%v]", elem, result) + } + } + if len(elems) != result.Size() { + t.Fatalf("expected cpuset %s to have the same size as %v", result, elems) + } +} + +func TestCPUSetSize(t *testing.T) { + testCases := []struct { + cpuset CPUSet + expected int + }{ + {NewCPUSet(), 0}, + {NewCPUSet(5), 1}, + {NewCPUSet(1, 2, 3, 4, 5), 5}, + } + + for _, c := range testCases { + actual := c.cpuset.Size() + if actual != c.expected { + t.Fatalf("expected: %d, actual: %d, cpuset: [%v]", c.expected, actual, c.cpuset) + } + } +} + +func TestCPUSetIsEmpty(t *testing.T) { + testCases := []struct { + cpuset CPUSet + expected bool + }{ + {NewCPUSet(), true}, + {NewCPUSet(5), false}, + {NewCPUSet(1, 2, 3, 4, 5), false}, + } + + for _, c := range testCases { + actual := c.cpuset.IsEmpty() + if actual != c.expected { + t.Fatalf("expected: %t, IsEmpty() returned: %t, cpuset: [%v]", c.expected, actual, c.cpuset) + } + } +} + +func TestCPUSetContains(t *testing.T) { + testCases := []struct { + cpuset CPUSet + mustContain []int + mustNotContain []int + }{ + {NewCPUSet(), []int{}, []int{1, 2, 3, 4, 5}}, + {NewCPUSet(5), []int{5}, []int{1, 2, 3, 4}}, + {NewCPUSet(1, 2, 4, 5), []int{1, 2, 4, 5}, []int{0, 3, 6}}, + } + + for _, c := range testCases { + for _, elem := range c.mustContain { + if !c.cpuset.Contains(elem) { + t.Fatalf("expected cpuset to contain element %d: [%v]", elem, c.cpuset) + } + } + for _, elem := range c.mustNotContain { + if c.cpuset.Contains(elem) { + t.Fatalf("expected cpuset not to contain element %d: [%v]", elem, c.cpuset) + } + } + } +} + +func TestCPUSetEqual(t *testing.T) { + shouldEqual := []struct { + s1 CPUSet + s2 CPUSet + }{ + {NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + } + + shouldNotEqual := []struct { + s1 CPUSet + s2 CPUSet + }{ + {NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)}, + } + + for _, c := range shouldEqual { + if !c.s1.Equals(c.s2) { + t.Fatalf("expected cpusets to be equal: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } + for _, c := range shouldNotEqual { + if c.s1.Equals(c.s2) { + t.Fatalf("expected cpusets to not be equal: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } +} + +func TestCPUSetIsSubsetOf(t *testing.T) { + shouldBeSubset := []struct { + s1 CPUSet + s2 CPUSet + }{ + // A set is a subset of itself + {NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + // Empty set is a subset of every set + {NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(2, 3), NewCPUSet(1, 2, 3, 4, 5)}, + } + + shouldNotBeSubset := []struct { + s1 CPUSet + s2 CPUSet + }{} + + for _, c := range shouldBeSubset { + if !c.s1.IsSubsetOf(c.s2) { + t.Fatalf("expected s1 to be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } + for _, c := range shouldNotBeSubset { + if c.s1.IsSubsetOf(c.s2) { + t.Fatalf("expected s1 to not be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2) + } + } +} + +func TestCPUSetUnionAll(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + s3 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(4), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 5), NewCPUSet(1, 2, 3, 4, 5)}, + } + for _, c := range testCases { + s := []CPUSet{} + s = append(s, c.s2) + s = append(s, c.s3) + result := c.s1.UnionAll(s) + if !result.Equals(c.expected) { + t.Fatalf("expected the union of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetUnion(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + } + + for _, c := range testCases { + result := c.s1.Union(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the union of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetIntersection(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(5)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(3)}, + } + + for _, c := range testCases { + result := c.s1.Intersection(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the intersection of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetDifference(t *testing.T) { + testCases := []struct { + s1 CPUSet + s2 CPUSet + expected CPUSet + }{ + {NewCPUSet(), NewCPUSet(), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(5), NewCPUSet()}, + {NewCPUSet(5), NewCPUSet(), NewCPUSet(5)}, + {NewCPUSet(5), NewCPUSet(5), NewCPUSet()}, + + {NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + + {NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()}, + {NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4)}, + + {NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)}, + {NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)}, + } + + for _, c := range testCases { + result := c.s1.Difference(c.s2) + if !result.Equals(c.expected) { + t.Fatalf("expected the difference of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2) + } + } +} + +func TestCPUSetToSlice(t *testing.T) { + testCases := []struct { + set CPUSet + expected []int + }{ + {NewCPUSet(), []int{}}, + {NewCPUSet(5), []int{5}}, + {NewCPUSet(1, 2, 3, 4, 5), []int{1, 2, 3, 4, 5}}, + } + + for _, c := range testCases { + result := c.set.ToSlice() + if !reflect.DeepEqual(result, c.expected) { + t.Fatalf("expected set as slice to be [%v] (got [%v]), s: [%v]", c.expected, result, c.set) + } + } +} + +func TestCPUSetString(t *testing.T) { + testCases := []struct { + set CPUSet + expected string + }{ + {NewCPUSet(), ""}, + {NewCPUSet(5), "5"}, + {NewCPUSet(1, 2, 3, 4, 5), "1-5"}, + {NewCPUSet(1, 2, 3, 5, 6, 8), "1-3,5-6,8"}, + } + + for _, c := range testCases { + result := c.set.String() + if result != c.expected { + t.Fatalf("expected set as string to be %s (got \"%s\"), s: [%v]", c.expected, result, c.set) + } + } +} + +func TestParse(t *testing.T) { + testCases := []struct { + cpusetString string + expected CPUSet + }{ + {"", NewCPUSet()}, + {"5", NewCPUSet(5)}, + {"1,2,3,4,5", NewCPUSet(1, 2, 3, 4, 5)}, + {"1-5", NewCPUSet(1, 2, 3, 4, 5)}, + {"1-2,3-5", NewCPUSet(1, 2, 3, 4, 5)}, + } + + for _, c := range testCases { + result, err := Parse(c.cpusetString) + if err != nil { + t.Fatalf("expected error not to have occurred: %v", err) + } + if !result.Equals(c.expected) { + t.Fatalf("expected string \"%s\" to parse as [%v] (got [%v])", c.cpusetString, c.expected, result) + } + } +} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 73ea8a95a3..8ac388e5dc 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -39,6 +39,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" vccgroups "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cpuset" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" From 64a2ef62e08213220c44cf38bf466be6e770b3f4 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 16:56:01 -0700 Subject: [PATCH 063/124] virtcontainers: add method for calculating cpuset for sandbox Calculate sandbox's CPUSet as the union of each of the container's CPUSets. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/sandbox.go | 22 +++++ src/runtime/virtcontainers/sandbox_test.go | 106 +++++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 8ac388e5dc..0117285c62 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -44,6 +44,7 @@ import ( vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) const ( @@ -2303,3 +2304,24 @@ func (s *Sandbox) GetOOMEvent() (string, error) { func (s *Sandbox) GetAgentURL() (string, error) { return s.agent.getAgentURL() } + +// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets +// as a string in canonical linux CPU list format +func (s *Sandbox) getSandboxCPUSet() (string, error) { + if s.config == nil { + return "", nil + } + + result := cpuset.NewCPUSet() + for _, ctr := range s.config.Containers { + if ctr.Resources.CPU != nil { + currSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) + if err != nil { + return "", fmt.Errorf("unable to parse CPUset for container %s", ctr.ID) + } + result = result.Union(currSet) + } + } + + return result.String(), nil +} diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 4af47ffaec..8345236f0f 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -1419,3 +1419,109 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) { }) } } + +func getContainerConfigWithCPUSet(cpuset string) ContainerConfig { + return ContainerConfig{ + Resources: specs.LinuxResources{ + CPU: &specs.LinuxCPU{ + Cpus: cpuset, + }, + }, + } +} + +func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox { + sandbox := Sandbox{} + + sandbox.config = &SandboxConfig{ + Containers: []ContainerConfig{ + getContainerConfigWithCPUSet(cpuset0), + getContainerConfigWithCPUSet(cpuset1), + getContainerConfigWithCPUSet(cpuset2), + }, + } + + return &sandbox +} + +func TestGetSandboxCpuSet(t *testing.T) { + + tests := []struct { + name string + cpuset0 string + cpuset1 string + cpuset2 string + result string + wantErr bool + }{ + { + "single, no cpuset", + "", + "", + "", + "", + false, + }, + { + "single cpuset", + "0", + "", + "", + "0", + false, + }, + { + "two duplicate cpuset", + "0", + "0", + "", + "0", + false, + }, + { + "3 cpusets", + "0-3", + "5-7", + "1", + "0-3,5-7", + false, + }, + { + "weird, but should be okay", + "0-3", + "99999", + "", + "0-3,99999", + false, + }, + { + "two, overlapping cpuset", + "0-3", + "1-2", + "", + "0-3", + false, + }, + { + "garbage, should fail", + "7 beard-seconds", + "Audrey + 7", + "Elliott - 17", + "", + true, + }, + } + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + s := getSimpleSandbox(tt.cpuset0, tt.cpuset1, tt.cpuset2) + res, err := s.getSandboxCPUSet() + if (err != nil) != tt.wantErr { + t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr) + } + if res != tt.result { + t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.result) + } + }) + } +} From 9bb0d48d56121b83c220c9513394d9f1c09a7b61 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 21:15:21 -0700 Subject: [PATCH 064/124] cpuset: support setting mems for sandbox CPUSet cgroup allows for pinning the memory associated with a cpuset to a given numa node. Similar to cpuset.cpus, we should take cpuset.mems into account for the sandbox-cgroup that Kata creates. Signed-off-by: Eric Ernst --- .../virtcontainers/pkg/cgroups/manager.go | 3 +- src/runtime/virtcontainers/sandbox.go | 33 ++++---- src/runtime/virtcontainers/sandbox_test.go | 82 +++++++++++-------- 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/cgroups/manager.go b/src/runtime/virtcontainers/pkg/cgroups/manager.go index e0f81777a1..23ca1d8279 100644 --- a/src/runtime/virtcontainers/pkg/cgroups/manager.go +++ b/src/runtime/virtcontainers/pkg/cgroups/manager.go @@ -332,7 +332,7 @@ func (m *Manager) RemoveDevice(device string) error { return fmt.Errorf("device %v not found in the cgroup", device) } -func (m *Manager) SetCPUSet(cpuset string) error { +func (m *Manager) SetCPUSet(cpuset, memset string) error { cgroups, err := m.GetCgroups() if err != nil { return err @@ -340,6 +340,7 @@ func (m *Manager) SetCPUSet(cpuset string) error { m.Lock() cgroups.CpusetCpus = cpuset + cgroups.CpusetMems = memset m.Unlock() return m.Apply() diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 0117285c62..0c3b9ff5a3 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -1996,15 +1996,13 @@ func (s *Sandbox) cgroupsUpdate() error { // in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be // updated. if s.config.SandboxCgroupOnly { - cpuset, err := s.getSandboxCPUSet() + cpuset, memset, err := s.getSandboxCPUSet() if err != nil { return err } - if cpuset != "" { - if err := s.cgroupMgr.SetCPUSet(cpuset); err != nil { - return err - } + if err := s.cgroupMgr.SetCPUSet(cpuset, memset); err != nil { + return err } return nil @@ -2305,23 +2303,30 @@ func (s *Sandbox) GetAgentURL() (string, error) { return s.agent.getAgentURL() } -// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets -// as a string in canonical linux CPU list format -func (s *Sandbox) getSandboxCPUSet() (string, error) { +// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets' +// cpus and mems as a string in canonical linux CPU/mems list format +func (s *Sandbox) getSandboxCPUSet() (string, string, error) { if s.config == nil { - return "", nil + return "", "", nil } - result := cpuset.NewCPUSet() + cpuResult := cpuset.NewCPUSet() + memResult := cpuset.NewCPUSet() for _, ctr := range s.config.Containers { if ctr.Resources.CPU != nil { - currSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) + currCpuSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) if err != nil { - return "", fmt.Errorf("unable to parse CPUset for container %s", ctr.ID) + return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err) } - result = result.Union(currSet) + cpuResult = cpuResult.Union(currCpuSet) + + currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems) + if err != nil { + return "", "", fmt.Errorf("unable to parse CPUset.mems for container %s: %v", ctr.ID, err) + } + memResult = memResult.Union(currMemSet) } } - return result.String(), nil + return cpuResult.String(), memResult.String(), nil } diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 8345236f0f..09b762a078 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -1420,24 +1420,25 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) { } } -func getContainerConfigWithCPUSet(cpuset string) ContainerConfig { +func getContainerConfigWithCPUSet(cpuset, memset string) ContainerConfig { return ContainerConfig{ Resources: specs.LinuxResources{ CPU: &specs.LinuxCPU{ Cpus: cpuset, + Mems: memset, }, }, } } -func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox { +func getSimpleSandbox(cpusets, memsets [3]string) *Sandbox { sandbox := Sandbox{} sandbox.config = &SandboxConfig{ Containers: []ContainerConfig{ - getContainerConfigWithCPUSet(cpuset0), - getContainerConfigWithCPUSet(cpuset1), - getContainerConfigWithCPUSet(cpuset2), + getContainerConfigWithCPUSet(cpusets[0], memsets[0]), + getContainerConfigWithCPUSet(cpusets[1], memsets[1]), + getContainerConfigWithCPUSet(cpusets[2], memsets[2]), }, } @@ -1447,80 +1448,97 @@ func getSimpleSandbox(cpuset0, cpuset1, cpuset2 string) *Sandbox { func TestGetSandboxCpuSet(t *testing.T) { tests := []struct { - name string - cpuset0 string - cpuset1 string - cpuset2 string - result string - wantErr bool + name string + cpusets [3]string + memsets [3]string + cpuResult string + memResult string + wantErr bool }{ { "single, no cpuset", - "", - "", + [3]string{"", "", ""}, + [3]string{"", "", ""}, "", "", false, }, { "single cpuset", + [3]string{"0", "", ""}, + [3]string{"", "", ""}, "0", "", - "", - "0", false, }, { "two duplicate cpuset", - "0", + [3]string{"0", "0", ""}, + [3]string{"", "", ""}, "0", "", - "0", false, }, { "3 cpusets", - "0-3", - "5-7", - "1", + [3]string{"0-3", "5-7", "1"}, + [3]string{"", "", ""}, "0-3,5-7", + "", false, }, + { "weird, but should be okay", - "0-3", - "99999", - "", + [3]string{"0-3", "99999", ""}, + [3]string{"", "", ""}, "0-3,99999", + "", false, }, { "two, overlapping cpuset", + [3]string{"0-3", "1-2", ""}, + [3]string{"", "", ""}, "0-3", - "1-2", "", - "0-3", false, }, { "garbage, should fail", - "7 beard-seconds", - "Audrey + 7", - "Elliott - 17", + [3]string{"7 beard-seconds", "Audrey + 7", "Elliott - 17"}, + [3]string{"", "", ""}, + "", "", true, }, + { + "cpuset and memset", + [3]string{"0-3", "1-2", ""}, + [3]string{"0", "1", "0-1"}, + "0-3", + "0-1", + false, + }, + { + "memset", + [3]string{"0-3", "1-2", ""}, + [3]string{"0", "3", ""}, + "0-3", + "0,3", + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := getSimpleSandbox(tt.cpuset0, tt.cpuset1, tt.cpuset2) - res, err := s.getSandboxCPUSet() + s := getSimpleSandbox(tt.cpusets, tt.memsets) + res, _, err := s.getSandboxCPUSet() if (err != nil) != tt.wantErr { t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr) } - if res != tt.result { - t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.result) + if res != tt.cpuResult { + t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.cpuResult) } }) } From 5c21ec278cff52a4cd8579c3cf2e396e9c907076 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 21:15:55 -0700 Subject: [PATCH 065/124] sandbox: consider cpusets if quota is not enforced CPUSet cgroup allows for pinning the memory associated with a cpuset to a given numa node. Similar to cpuset.cpus, we should take cpuset.mems into account for the sandbox-cgroup that Kata creates. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/sandbox.go | 27 ++++++++++++++++++---- src/runtime/virtcontainers/sandbox_test.go | 15 +++++++++--- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 0c3b9ff5a3..4e4d6cb736 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -1898,7 +1898,10 @@ func (s *Sandbox) updateResources() error { return fmt.Errorf("sandbox config is nil") } - sandboxVCPUs := s.calculateSandboxCPUs() + sandboxVCPUs, err := s.calculateSandboxCPUs() + if err != nil { + return err + } // Add default vcpus for sandbox sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs @@ -1960,8 +1963,9 @@ func (s *Sandbox) calculateSandboxMemory() int64 { return memorySandbox } -func (s *Sandbox) calculateSandboxCPUs() uint32 { +func (s *Sandbox) calculateSandboxCPUs() (uint32, error) { mCPU := uint32(0) + cpusetCount := int(0) for _, c := range s.config.Containers { // Do not hot add again non-running containers resources @@ -1975,9 +1979,22 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 { mCPU += utils.CalculateMilliCPUs(*cpu.Quota, *cpu.Period) } + set, err := cpuset.Parse(cpu.Cpus) + if err != nil { + return 0, nil + } + cpusetCount += set.Size() } } - return utils.CalculateVCpusFromMilliCpus(mCPU) + + // If we aren't being constrained, then we could have two scenarios: + // 1. BestEffort QoS: no proper support today in Kata. + // 2. We could be constrained only by CPUSets. Check for this: + if mCPU == 0 && cpusetCount > 0 { + return uint32(cpusetCount), nil + } + + return utils.CalculateVCpusFromMilliCpus(mCPU), nil } // GetHypervisorType is used for getting Hypervisor name currently used. @@ -2314,11 +2331,11 @@ func (s *Sandbox) getSandboxCPUSet() (string, string, error) { memResult := cpuset.NewCPUSet() for _, ctr := range s.config.Containers { if ctr.Resources.CPU != nil { - currCpuSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) + currCPUSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus) if err != nil { return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err) } - cpuResult = cpuResult.Union(currCpuSet) + cpuResult = cpuResult.Union(currCPUSet) currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems) if err != nil { diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 09b762a078..6e6dd752c1 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -106,12 +106,18 @@ func TestCreateMockSandbox(t *testing.T) { func TestCalculateSandboxCPUs(t *testing.T) { sandbox := &Sandbox{} sandbox.config = &SandboxConfig{} + unconstrained := newTestContainerConfigNoop("cont-00001") - constrained := newTestContainerConfigNoop("cont-00001") + constrained := newTestContainerConfigNoop("cont-00002") + unconstrainedCpusets0_1 := newTestContainerConfigNoop("cont-00003") + unconstrainedCpusets2 := newTestContainerConfigNoop("cont-00004") + constrainedCpusets0_7 := newTestContainerConfigNoop("cont-00005") quota := int64(4000) period := uint64(1000) constrained.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: "a} - + unconstrainedCpusets0_1.Resources.CPU = &specs.LinuxCPU{Cpus: "0-1"} + unconstrainedCpusets2.Resources.CPU = &specs.LinuxCPU{Cpus: "2"} + constrainedCpusets0_7.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: "a, Cpus: "0-7"} tests := []struct { name string containers []ContainerConfig @@ -123,11 +129,14 @@ func TestCalculateSandboxCPUs(t *testing.T) { {"2-constrained", []ContainerConfig{constrained, constrained}, 8}, {"3-mix-constraints", []ContainerConfig{unconstrained, constrained, constrained}, 8}, {"3-constrained", []ContainerConfig{constrained, constrained, constrained}, 12}, + {"unconstrained-1-cpuset", []ContainerConfig{unconstrained, unconstrained, unconstrainedCpusets0_1}, 2}, + {"unconstrained-2-cpuset", []ContainerConfig{unconstrainedCpusets0_1, unconstrainedCpusets2}, 3}, + {"constrained-cpuset", []ContainerConfig{constrainedCpusets0_7}, 4}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sandbox.config.Containers = tt.containers - got := sandbox.calculateSandboxCPUs() + got, _ := sandbox.calculateSandboxCPUs() assert.Equal(t, got, tt.want) }) } From bfbbe8ba6b33292bc0c5b2a5037afc514ec2b0ea Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 21:18:08 -0700 Subject: [PATCH 066/124] cpuset: don't set cpuset.mems in the guest Kata doesn't map any numa topologies in the guest. Let's make sure we clear the Cpuset fields before passing container updates to the guest. Note, in the future we may want to have a vCPU to guest CPU mapping and still include the cpuset.Cpus. Until we have this support, clear this as well. Fixes: #932 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/container.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 2c49b14bc5..21f18f35d7 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -1165,6 +1165,14 @@ func (c *Container) update(resources specs.LinuxResources) error { } } + // There currently isn't a notion of cpusets.cpus or mems being tracked + // inside of the guest. Make sure we clear these before asking agent to update + // the container's cgroups. + if resources.CPU != nil { + resources.CPU.Mems = "" + resources.CPU.Cpus = "" + } + return c.sandbox.agent.updateContainer(c.sandbox, *c, resources) } From 183823398d1f231f7380c1b170e810d51c540ead Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 12 Oct 2020 18:10:27 -0700 Subject: [PATCH 067/124] cpuset: add cpuset pkg Pulled from 1.18.4 Kubernetes, adding the cpuset pkg for managing CPUSet calculations on the host. Go mod'ing the original code from k8s.io/kubernetes was very painful, and this is very static, so let's just pull in what we need. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/sandbox.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 4e4d6cb736..16db127ea1 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -44,7 +44,6 @@ import ( vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" - "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) const ( From b33d4fe7080f1af407194aece356e7f04a1b52da Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Wed, 14 Oct 2020 12:50:38 +0800 Subject: [PATCH 068/124] agent: fix panic on malformed device resource in container update Somehow containerd is sending a malformed device in update API. While it should not happen, we should not panic either. Fixes: #946 Signed-off-by: Peng Tao --- src/agent/rustjail/src/cgroups/fs/mod.rs | 37 +++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 1b70a28edf..c7fcee8c89 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -266,18 +266,21 @@ fn set_devices_resources( let mut devices = vec![]; for d in device_resources.iter() { - let dev = linux_device_group_to_cgroup_device(&d); - devices.push(dev); + if let Some(dev) = linux_device_group_to_cgroup_device(&d) { + devices.push(dev); + } } for d in DEFAULT_DEVICES.iter() { - let dev = linux_device_to_cgroup_device(&d); - devices.push(dev); + if let Some(dev) = linux_device_to_cgroup_device(&d) { + devices.push(dev); + } } for d in DEFAULT_ALLOWED_DEVICES.iter() { - let dev = linux_device_group_to_cgroup_device(&d); - devices.push(dev); + if let Some(dev) = linux_device_group_to_cgroup_device(&d) { + devices.push(dev); + } } res.devices.update_values = true; @@ -465,8 +468,11 @@ fn build_blk_io_device_throttle_resource( blk_io_device_throttle_resources } -fn linux_device_to_cgroup_device(d: &LinuxDevice) -> DeviceResource { - let dev_type = DeviceType::from_char(d.r#type.chars().next()).unwrap(); +fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option { + let dev_type = match DeviceType::from_char(d.r#type.chars().next()) { + Some(t) => t, + None => return None, + }; let permissions = vec![ DevicePermissions::Read, @@ -474,17 +480,20 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> DeviceResource { DevicePermissions::MkNod, ]; - DeviceResource { + Some(DeviceResource { allow: true, devtype: dev_type, major: d.major, minor: d.minor, access: permissions, - } + }) } -fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> DeviceResource { - let dev_type = DeviceType::from_char(d.r#type.chars().next()).unwrap(); +fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option { + let dev_type = match DeviceType::from_char(d.r#type.chars().next()) { + Some(t) => t, + None => return None, + }; let mut permissions: Vec = vec![]; for p in d.access.chars().collect::>() { @@ -496,13 +505,13 @@ fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> DeviceResource } } - DeviceResource { + Some(DeviceResource { allow: d.allow, devtype: dev_type, major: d.major.unwrap_or(0), minor: d.minor.unwrap_or(0), access: permissions, - } + }) } // split space separated values into an vector of u64 From 9e5ed41511f5964fe4448a36387650038457283c Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Wed, 13 May 2020 16:58:55 +0200 Subject: [PATCH 069/124] config: Add 'List' alternates for hypervisor configuration paths Paths mentioned in the hypervisor configuration can be overriden using annotations, which is potentially dangerous. For each path, add a 'List' variant that specifies the list of acceptable values from annotations. Bug: https://bugs.launchpad.net/katacontainers.io/+bug/1878234 Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/pkg/katautils/config.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e967b081e9..71f9ec715b 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -71,9 +71,12 @@ type factory struct { type hypervisor struct { Path string `toml:"path"` + PathList []string `toml:"path_list"` JailerPath string `toml:"jailer_path"` + JailerPathList []string `toml:"jailer_path_list"` Kernel string `toml:"kernel"` CtlPath string `toml:"ctlpath"` + CtlPathList []string `toml:"ctlpath_list"` Initrd string `toml:"initrd"` Image string `toml:"image"` Firmware string `toml:"firmware"` @@ -85,6 +88,7 @@ type hypervisor struct { EntropySource string `toml:"entropy_source"` SharedFS string `toml:"shared_fs"` VirtioFSDaemon string `toml:"virtio_fs_daemon"` + VirtioFSDaemonList []string `toml:"virtio_fs_daemon_list"` VirtioFSCache string `toml:"virtio_fs_cache"` VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` @@ -93,6 +97,7 @@ type hypervisor struct { BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` EnableVhostUserStore bool `toml:"enable_vhost_user_store"` VhostUserStorePath string `toml:"vhost_user_store_path"` + VhostUserStorePathList []string `toml:"vhost_user_store_path_list"` NumVCPUs int32 `toml:"default_vcpus"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` @@ -108,6 +113,7 @@ type hypervisor struct { IOMMU bool `toml:"enable_iommu"` IOMMUPlatform bool `toml:"enable_iommu_platform"` FileBackedMemRootDir string `toml:"file_mem_backend"` + FileBackedMemRootList []string `toml:"file_mem_backend_list"` Swap bool `toml:"enable_swap"` Debug bool `toml:"enable_debug"` DisableNestingChecks bool `toml:"disable_nesting_checks"` @@ -647,6 +653,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableBlockDeviceUse: h.DisableBlockDeviceUse, SharedFS: sharedFS, VirtioFSDaemon: h.VirtioFSDaemon, + VirtioFSDaemonList: h.VirtioFSDaemonList, VirtioFSCacheSize: h.VirtioFSCacheSize, VirtioFSCache: h.defaultVirtioFSCache(), VirtioFSExtraArgs: h.VirtioFSExtraArgs, From 2faafbdd3a55ee822a8b97c53ec358b1f746b986 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 14 May 2020 20:18:18 +0200 Subject: [PATCH 070/124] config: Protect virtio_fs_daemon annotation Sending the virtio_fs_daemon annotation can be used to execute arbitrary code on the host. In order to prevent this, restrict the values of the annotation to a list provided by the configuration file. Fixes: #901 Signed-off-by: Christophe de Dinechin --- .../cli/config/configuration-clh.toml.in | 3 ++ .../configuration-qemu-virtiofs.toml.in | 5 ++- .../cli/config/configuration-qemu.toml.in | 17 +++++--- src/runtime/pkg/katautils/config.go | 7 ++- src/runtime/virtcontainers/hypervisor.go | 6 +++ src/runtime/virtcontainers/persist.go | 4 ++ .../virtcontainers/persist/api/config.go | 6 +++ src/runtime/virtcontainers/pkg/oci/utils.go | 29 +++++++++---- .../virtcontainers/pkg/oci/utils_test.go | 43 +++++++++++++++---- 9 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index d4d4e7978d..b3a74705ac 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -62,6 +62,9 @@ default_memory = @DEFMEMSZ@ # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" +# List of valid annotations values for the virtiofs daemon (default: empty) +# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] + # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index a199bd30df..4a1523b439 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -110,6 +110,9 @@ shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" +# List of valid annotations values for the virtiofs daemon (default: empty) +# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] + # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -238,7 +241,7 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" #hotplug_vfio_on_root_bus = true # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off -# security (vhost-net runs ring0) for network I/O performance. +# security (vhost-net runs ring0) for network I/O performance. #disable_vhost_net = true # diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 0da8d50666..22054c0374 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -101,10 +101,10 @@ default_memory = @DEFMEMSZ@ #enable_virtio_mem = true # Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's +# In case of a storage driver like devicemapper where a container's # root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, +# directly to the hypervisor for performance reasons. +# This flag prevents the block device from being passed to the hypervisor, # 9pfs is used instead to pass the rootfs. disable_block_device_use = @DEFDISABLEBLOCK@ @@ -116,6 +116,9 @@ shared_fs = "@DEFSHAREDFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" +# List of valid annotations values for the virtiofs daemon (default: empty) +# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] + # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -180,7 +183,7 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Enabling this will result in the VM memory # being allocated using huge pages. # This is useful when you want to use vhost-user network -# stacks within the container. This will automatically +# stacks within the container. This will automatically # result in memory pre allocation #enable_hugepages = true @@ -236,9 +239,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Default is false #disable_image_nvdimm = true -# VFIO devices are hotplugged on a bridge by default. +# VFIO devices are hotplugged on a bridge by default. # Enable hotplugging on root bus. This may be required for devices with -# a large PCI bar, as this is a current limitation with hotplugging on +# a large PCI bar, as this is a current limitation with hotplugging on # a bridge. This value is valid for "pc" machine type. # Default false #hotplug_vfio_on_root_bus = true @@ -251,7 +254,7 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" #pcie_root_port = 2 # If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off -# security (vhost-net runs ring0) for network I/O performance. +# security (vhost-net runs ring0) for network I/O performance. #disable_vhost_net = true # diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 71f9ec715b..5a5e45a00f 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -71,7 +71,7 @@ type factory struct { type hypervisor struct { Path string `toml:"path"` - PathList []string `toml:"path_list"` + HypervisorPathList []string `toml:"path_list"` JailerPath string `toml:"jailer_path"` JailerPathList []string `toml:"jailer_path_list"` Kernel string `toml:"kernel"` @@ -533,6 +533,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{ HypervisorPath: hypervisor, + HypervisorPathList: h.HypervisorPathList, JailerPath: jailer, KernelPath: kernel, InitrdPath: initrd, @@ -634,6 +635,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{ HypervisorPath: hypervisor, + HypervisorPathList: h.HypervisorPathList, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, @@ -723,6 +725,7 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{ HypervisorPath: hypervisor, + HypervisorPathList: h.HypervisorPathList, KernelPath: kernel, ImagePath: image, HypervisorCtlPath: hypervisorctl, @@ -793,6 +796,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{ HypervisorPath: hypervisor, + HypervisorPathList: h.HypervisorPathList, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, @@ -811,6 +815,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableBlockDeviceUse: h.DisableBlockDeviceUse, SharedFS: sharedFS, VirtioFSDaemon: h.VirtioFSDaemon, + VirtioFSDaemonList: h.VirtioFSDaemonList, VirtioFSCacheSize: h.VirtioFSCacheSize, VirtioFSCache: h.VirtioFSCache, MemPrealloc: h.MemPrealloc, diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index ccf9434cfd..a7371c2a33 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -275,6 +275,9 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorPathList is the list of hypervisor paths names allowed in annotations + HypervisorPathList []string + // HypervisorCtlPath is the hypervisor ctl executable host path. HypervisorCtlPath string @@ -309,6 +312,9 @@ type HypervisorConfig struct { // VirtioFSDaemon is the virtio-fs vhost-user daemon path VirtioFSDaemon string + // VirtioFSDaemonList is the list of valid virtiofs names for annotations + VirtioFSDaemonList []string + // VirtioFSCache cache mode for fs version cache or "none" VirtioFSCache string diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 4a1962a12d..ef762ba528 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -212,6 +212,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { MachineAccelerators: sconfig.HypervisorConfig.MachineAccelerators, CPUFeatures: sconfig.HypervisorConfig.CPUFeatures, HypervisorPath: sconfig.HypervisorConfig.HypervisorPath, + HypervisorPathList: sconfig.HypervisorConfig.HypervisorPathList, HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, JailerPath: sconfig.HypervisorConfig.JailerPath, BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, @@ -221,6 +222,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { EntropySource: sconfig.HypervisorConfig.EntropySource, SharedFS: sconfig.HypervisorConfig.SharedFS, VirtioFSDaemon: sconfig.HypervisorConfig.VirtioFSDaemon, + VirtioFSDaemonList: sconfig.HypervisorConfig.VirtioFSDaemonList, VirtioFSCache: sconfig.HypervisorConfig.VirtioFSCache, VirtioFSExtraArgs: sconfig.HypervisorConfig.VirtioFSExtraArgs[:], BlockDeviceCacheSet: sconfig.HypervisorConfig.BlockDeviceCacheSet, @@ -473,6 +475,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { MachineAccelerators: hconf.MachineAccelerators, CPUFeatures: hconf.CPUFeatures, HypervisorPath: hconf.HypervisorPath, + HypervisorPathList: hconf.HypervisorPathList, HypervisorCtlPath: hconf.HypervisorCtlPath, JailerPath: hconf.JailerPath, BlockDeviceDriver: hconf.BlockDeviceDriver, @@ -482,6 +485,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { EntropySource: hconf.EntropySource, SharedFS: hconf.SharedFS, VirtioFSDaemon: hconf.VirtioFSDaemon, + VirtioFSDaemonList: hconf.VirtioFSDaemonList, VirtioFSCache: hconf.VirtioFSCache, VirtioFSExtraArgs: hconf.VirtioFSExtraArgs[:], BlockDeviceCacheSet: hconf.BlockDeviceCacheSet, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 5bf45ff71a..1db4833025 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -60,6 +60,9 @@ type HypervisorConfig struct { // HypervisorPath is the hypervisor executable host path. HypervisorPath string + // HypervisorPathList is the list of hypervisor paths names allowed in annotations + HypervisorPathList []string + // HypervisorCtlPath is the hypervisor ctl executable host path. HypervisorCtlPath string @@ -94,6 +97,9 @@ type HypervisorConfig struct { // VirtioFSDaemon is the virtio-fs vhost-user daemon path VirtioFSDaemon string + // VirtioFSDaemonList is the list of valid virtiofs names for annotations + VirtioFSDaemonList []string + // VirtioFSCache cache mode for fs version cache or "none" VirtioFSCache string diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 1562c5bab7..779147c1b3 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "path/filepath" + "regexp" goruntime "runtime" "strconv" "strings" @@ -190,6 +191,15 @@ func contains(s []string, e string) bool { return false } +func regexpContains(s []string, e string) bool { + for _, a := range s { + if matched, _ := regexp.MatchString(a, e); matched { + return true + } + } + return false +} + func newLinuxDeviceInfo(d specs.LinuxDevice) (*config.DeviceInfo, error) { allowedDeviceTypes := []string{"c", "b", "u", "p"} @@ -322,13 +332,13 @@ func SandboxID(spec specs.Spec) (string, error) { return "", fmt.Errorf("Could not find sandbox ID") } -func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { +func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig, runtime RuntimeConfig) error { addAssetAnnotations(ocispec, config) - if err := addHypervisorConfigOverrides(ocispec, config); err != nil { + if err := addHypervisorConfigOverrides(ocispec, config, runtime); err != nil { return err } - if err := addRuntimeConfigOverrides(ocispec, config); err != nil { + if err := addRuntimeConfigOverrides(ocispec, config, runtime); err != nil { return err } @@ -361,7 +371,7 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { } } -func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) error { +func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, runtime RuntimeConfig) error { if err := addHypervisorCPUOverrides(ocispec, config); err != nil { return err } @@ -374,7 +384,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) return err } - if err := addHypervisporVirtioFsOverrides(ocispec, config); err != nil { + if err := addHypervisporVirtioFsOverrides(ocispec, config, runtime); err != nil { return err } @@ -646,7 +656,7 @@ func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) return nil } -func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { +func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.SharedFS]; ok { supportedSharedFS := []string{config.Virtio9P, config.VirtioFS} valid := false @@ -663,6 +673,9 @@ func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxCon } if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok { + if !regexpContains(runtime.HypervisorConfig.VirtioFSDaemonList, value) { + return fmt.Errorf("virtiofs daemon %v required from annotation is not valid", value) + } sbConfig.HypervisorConfig.VirtioFSDaemon = value } @@ -730,7 +743,7 @@ func addHypervisporNetworkOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConf return nil } -func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { +func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DisableGuestSeccomp]; ok { disableGuestSeccomp, err := strconv.ParseBool(value) if err != nil { @@ -870,7 +883,7 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c Experimental: runtime.Experimental, } - if err := addAnnotations(ocispec, &sandboxConfig); err != nil { + if err := addAnnotations(ocispec, &sandboxConfig, runtime); err != nil { return vc.SandboxConfig{}, err } diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index 8b011f5bd1..533ee2ec71 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -676,7 +676,12 @@ func TestAddAssetAnnotations(t *testing.T) { Annotations: expectedAnnotations, } - addAnnotations(ocispec, &config) + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + + addAnnotations(ocispec, &config, runtimeConfig) assert.Exactly(expectedAnnotations, config.Annotations) } @@ -700,9 +705,14 @@ func TestAddAgentAnnotations(t *testing.T) { ContainerPipeSize: 1024, } + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + ocispec.Annotations[vcAnnotations.KernelModules] = strings.Join(expectedAgentConfig.KernelModules, KernelModulesSeparator) ocispec.Annotations[vcAnnotations.AgentContainerPipeSize] = "1024" - addAnnotations(ocispec, &config) + addAnnotations(ocispec, &config, runtimeConfig) assert.Exactly(expectedAgentConfig, config.AgentConfig) } @@ -722,8 +732,13 @@ func TestContainerPipeSizeAnnotation(t *testing.T) { ContainerPipeSize: 0, } + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + ocispec.Annotations[vcAnnotations.AgentContainerPipeSize] = "foo" - err := addAnnotations(ocispec, &config) + err := addAnnotations(ocispec, &config, runtimeConfig) assert.Error(err) assert.Exactly(expectedAgentConfig, config.AgentConfig) } @@ -752,8 +767,13 @@ func TestAddHypervisorAnnotations(t *testing.T) { }, } + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" - addHypervisorConfigOverrides(ocispec, &config) + addHypervisorConfigOverrides(ocispec, &config, runtimeConfig) assert.Exactly(expectedHyperConfig, config.HypervisorConfig) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" @@ -791,7 +811,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000" ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000" - addAnnotations(ocispec, &config) + addAnnotations(ocispec, &config, runtimeConfig) assert.Equal(config.HypervisorConfig.NumVCPUs, uint32(1)) assert.Equal(config.HypervisorConfig.DefaultMaxVCPUs, uint32(1)) assert.Equal(config.HypervisorConfig.MemorySize, uint32(1024)) @@ -828,16 +848,16 @@ func TestAddHypervisorAnnotations(t *testing.T) { // In case an absurd large value is provided, the config value if not over-ridden ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536" - err := addAnnotations(ocispec, &config) + err := addAnnotations(ocispec, &config, runtimeConfig) assert.Error(err) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1" - err = addAnnotations(ocispec, &config) + err = addAnnotations(ocispec, &config, runtimeConfig) assert.Error(err) ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-1" - err = addAnnotations(ocispec, &config) + err = addAnnotations(ocispec, &config, runtimeConfig) assert.Error(err) ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "1" @@ -856,12 +876,17 @@ func TestAddRuntimeAnnotations(t *testing.T) { Annotations: make(map[string]string), } + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + ocispec.Annotations[vcAnnotations.DisableGuestSeccomp] = "true" ocispec.Annotations[vcAnnotations.SandboxCgroupOnly] = "true" ocispec.Annotations[vcAnnotations.DisableNewNetNs] = "true" ocispec.Annotations[vcAnnotations.InterNetworkModel] = "macvtap" - addAnnotations(ocispec, &config) + addAnnotations(ocispec, &config, runtimeConfig) assert.Equal(config.DisableGuestSeccomp, true) assert.Equal(config.SandboxCgroupOnly, true) assert.Equal(config.NetworkConfig.DisableNewNetNs, true) From 2f5f35608ac8d9c33e46c9071adee496490ef2db Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 15:59:30 +0200 Subject: [PATCH 071/124] config: Fix typo in function name There was an extra 'p' in addHypervisorVirtioFsOverrides. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 779147c1b3..c4bfaaf554 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -384,7 +384,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return err } - if err := addHypervisporVirtioFsOverrides(ocispec, config, runtime); err != nil { + if err := addHypervisorVirtioFsOverrides(ocispec, config, runtime); err != nil { return err } @@ -656,7 +656,7 @@ func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) return nil } -func addHypervisporVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { +func addHypervisorVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.SharedFS]; ok { supportedSharedFS := []string{config.Virtio9P, config.VirtioFS} valid := false From 80144fc4153cdf0e35c4712d3058e1a4587cb5f8 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 16:04:55 +0200 Subject: [PATCH 072/124] config: Add hypervisor path override through annotations The annotation is provided, so it should be respected. Furthermore, it is important to implement it with the appropriate protetions similar to what was done for virtiofsd. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index c4bfaaf554..8d0d0f9606 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -392,6 +392,13 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return err } + if value, ok := ocispec.Annotations[vcAnnotations.HypervisorPath]; ok { + if !regexpContains(runtime.HypervisorConfig.HypervisorPathList, value) { + return fmt.Errorf("hypervisor %v required from annotation is not valid", value) + } + config.HypervisorConfig.HypervisorPath = value + } + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { if value != "" { params := vc.DeserializeParams(strings.Fields(value)) From 3f7bcf54f0596e1b8d895b19f47d9f6dfd0d5e6b Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 16:41:04 +0200 Subject: [PATCH 073/124] annotations: Simplify negative logic Replace strange negative logic (!ok -> continue) with positive logic (ok -> do it) Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 8d0d0f9606..a96f920b89 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -363,11 +363,9 @@ func addAssetAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) { for _, a := range assetAnnotations { value, ok := ocispec.Annotations[a] - if !ok { - continue + if ok { + config.Annotations[a] = value } - - config.Annotations[a] = value } } From fe5e1cf2e1c0cb8789cab625f42f5bc30add9c23 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 17:40:43 +0200 Subject: [PATCH 074/124] config: Add examples for path_list configuration The path_list configuration gives a series of regular expressions that limit which values are acceptable through annotations in order to avoid kata launching arbitrary binaries on the host when receiving an annotation. Fixes: #901 Signed-off-by: Christophe de Dinechin --- .../cli/config/configuration-acrn.toml.in | 4 ++++ .../cli/config/configuration-clh.toml.in | 3 +++ .../cli/config/configuration-fc.toml.in | 18 ++++++++++++------ .../config/configuration-qemu-virtiofs.toml.in | 4 ++++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 15fb00e45d..7bc8d3172e 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -16,6 +16,10 @@ ctlpath = "@ACRNCTLPATH@" kernel = "@KERNELPATH_ACRN@" image = "@IMAGEPATH@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression +# path_list = [ "@ACRNPATH@.*" ] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index b3a74705ac..2b9f1b6b58 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -12,6 +12,9 @@ [hypervisor.clh] path = "@CLHPATH@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression +# path_list = [ "@CLHPATH@.*" ] kernel = "@KERNELPATH_CLH@" image = "@IMAGEPATH@" diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index e72130e0ed..7aa638e849 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -12,6 +12,13 @@ [hypervisor.firecracker] path = "@FCPATH@" +kernel = "@KERNELPATH_FC@" +image = "@IMAGEPATH@" + +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression +# path_list = [ "@FCPATH@.*" ] + # Path for the jailer specific to firecracker # If the jailer path is not set kata will launch firecracker # without a jail. If the jailer is set firecracker will be @@ -19,8 +26,7 @@ path = "@FCPATH@" # This is disabled by default as additional setup is required # for this feature today. #jailer_path = "@FCJAILERPATH@" -kernel = "@KERNELPATH_FC@" -image = "@IMAGEPATH@" + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -87,10 +93,10 @@ default_memory = @DEFMEMSZ@ #memory_offset = 0 # Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's +# In case of a storage driver like devicemapper where a container's # root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, +# directly to the hypervisor for performance reasons. +# This flag prevents the block device from being passed to the hypervisor, # 9pfs is used instead to pass the rootfs. disable_block_device_use = @DEFDISABLEBLOCK@ @@ -126,7 +132,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@" # Enabling this will result in the VM memory # being allocated using huge pages. # This is useful when you want to use vhost-user network -# stacks within the container. This will automatically +# stacks within the container. This will automatically # result in memory pre allocation #enable_hugepages = true diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 4a1523b439..cc0c63e483 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -16,6 +16,10 @@ kernel = "@KERNELVIRTIOFSPATH@" image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression +# path_list = [ "@QEMUPATH@.*" ] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. From 2d65b3bfd85c33c29d6d1d574badebf70e0453f7 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 17:49:01 +0200 Subject: [PATCH 075/124] config: Protect jailer_path annotation The jailer_path annotation can be used to execute arbitrary code on the host. Add a jailer_path_list configuration entry providing a list of regular expressions that can be used to filter annotations that represent valid file names. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/cli/config/configuration-fc.toml.in | 4 ++++ src/runtime/pkg/katautils/config.go | 1 + src/runtime/virtcontainers/hypervisor.go | 3 +++ src/runtime/virtcontainers/persist.go | 2 ++ src/runtime/virtcontainers/persist/api/config.go | 3 +++ src/runtime/virtcontainers/pkg/oci/utils.go | 7 +++++++ 6 files changed, 20 insertions(+) diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 7aa638e849..f54b7ba4ae 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -27,6 +27,10 @@ image = "@IMAGEPATH@" # for this feature today. #jailer_path = "@FCJAILERPATH@" +# List of valid jailer path values for the hypervisor (default: empty) +# Each member of the list can be a regular expression +# jailer_path_list = [ "@FCJAILERPATH@.*" ] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 5a5e45a00f..17f16fa92d 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -535,6 +535,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { HypervisorPath: hypervisor, HypervisorPathList: h.HypervisorPathList, JailerPath: jailer, + JailerPathList: h.JailerPathList, KernelPath: kernel, InitrdPath: initrd, ImagePath: image, diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index a7371c2a33..1c65c16d8c 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -284,6 +284,9 @@ type HypervisorConfig struct { // JailerPath is the jailer executable host path. JailerPath string + // JailerPathList is the list of jailer paths names allowed in annotations + JailerPathList []string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index ef762ba528..c02b0868a1 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { HypervisorPathList: sconfig.HypervisorConfig.HypervisorPathList, HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, JailerPath: sconfig.HypervisorConfig.JailerPath, + JailerPathList: sconfig.HypervisorConfig.JailerPathList, BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType, MemoryPath: sconfig.HypervisorConfig.MemoryPath, @@ -478,6 +479,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { HypervisorPathList: hconf.HypervisorPathList, HypervisorCtlPath: hconf.HypervisorCtlPath, JailerPath: hconf.JailerPath, + JailerPathList: hconf.JailerPathList, BlockDeviceDriver: hconf.BlockDeviceDriver, HypervisorMachineType: hconf.HypervisorMachineType, MemoryPath: hconf.MemoryPath, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 1db4833025..86b1f2565c 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -69,6 +69,9 @@ type HypervisorConfig struct { // JailerPath is the jailer executable host path. JailerPath string + // JailerPathList is the list of jailer paths names allowed in annotations + JailerPathList []string + // BlockDeviceDriver specifies the driver to be used for block device // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver BlockDeviceDriver string diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index a96f920b89..236ea47dc4 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -397,6 +397,13 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, config.HypervisorConfig.HypervisorPath = value } + if value, ok := ocispec.Annotations[vcAnnotations.JailerPath]; ok { + if !regexpContains(runtime.HypervisorConfig.JailerPathList, value) { + return fmt.Errorf("jailer %v required from annotation is not valid", value) + } + config.HypervisorConfig.JailerPath = value + } + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { if value != "" { params := vc.DeserializeParams(strings.Fields(value)) From b5f2a1e8c4d445999b32b1ddb41d5ed0006c0034 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 18:10:53 +0200 Subject: [PATCH 076/124] config: Protect ctlpath from annotation attack This also adds annotation for ctlpath which were not present before. It's better to implement the code consistenly right now to make sure that we don't end up with a leaky implementation tacked on later. Fixes: #901 Signed-off-by: Christophe de Dinechin --- .../cli/config/configuration-acrn.toml.in | 3 ++ src/runtime/pkg/katautils/config.go | 41 ++++++++++--------- src/runtime/virtcontainers/hypervisor.go | 3 ++ src/runtime/virtcontainers/persist.go | 2 + .../virtcontainers/persist/api/config.go | 4 ++ .../pkg/annotations/annotations.go | 3 ++ src/runtime/virtcontainers/pkg/oci/utils.go | 7 ++++ 7 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 7bc8d3172e..f6e9b7849e 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -20,6 +20,9 @@ image = "@IMAGEPATH@" # Each member of the list can be a regular expression # path_list = [ "@ACRNPATH@.*" ] +# List of valid annotations values for ctlpath (default: empty) +# ctlpath_list = [ "@ACRNCTLPATH@.*" ] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 17f16fa92d..adb398bd77 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -725,26 +725,27 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } return vc.HypervisorConfig{ - HypervisorPath: hypervisor, - HypervisorPathList: h.HypervisorPathList, - KernelPath: kernel, - ImagePath: image, - HypervisorCtlPath: hypervisorctl, - FirmwarePath: firmware, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), - NumVCPUs: h.defaultVCPUs(), - DefaultMaxVCPUs: h.defaultMaxVCPUs(), - MemorySize: h.defaultMemSz(), - MemSlots: h.defaultMemSlots(), - EntropySource: h.GetEntropySource(), - DefaultBridges: h.defaultBridges(), - HugePages: h.HugePages, - Mlock: !h.Swap, - Debug: h.Debug, - DisableNestingChecks: h.DisableNestingChecks, - BlockDeviceDriver: blockDriver, - DisableVhostNet: h.DisableVhostNet, - GuestHookPath: h.guestHookPath(), + HypervisorPath: hypervisor, + HypervisorPathList: h.HypervisorPathList, + KernelPath: kernel, + ImagePath: image, + HypervisorCtlPath: hypervisorctl, + HypervisorCtlPathList: h.CtlPathList, + FirmwarePath: firmware, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), + HugePages: h.HugePages, + Mlock: !h.Swap, + Debug: h.Debug, + DisableNestingChecks: h.DisableNestingChecks, + BlockDeviceDriver: blockDriver, + DisableVhostNet: h.DisableVhostNet, + GuestHookPath: h.guestHookPath(), }, nil } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 1c65c16d8c..a50fd5f5ab 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -278,6 +278,9 @@ type HypervisorConfig struct { // HypervisorPathList is the list of hypervisor paths names allowed in annotations HypervisorPathList []string + // HypervisorCtlPathList is the list of hypervisor control paths names allowed in annotations + HypervisorCtlPathList []string + // HypervisorCtlPath is the hypervisor ctl executable host path. HypervisorCtlPath string diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index c02b0868a1..9e6789fe7b 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { HypervisorPath: sconfig.HypervisorConfig.HypervisorPath, HypervisorPathList: sconfig.HypervisorConfig.HypervisorPathList, HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath, + HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList, JailerPath: sconfig.HypervisorConfig.JailerPath, JailerPathList: sconfig.HypervisorConfig.JailerPathList, BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, @@ -478,6 +479,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { HypervisorPath: hconf.HypervisorPath, HypervisorPathList: hconf.HypervisorPathList, HypervisorCtlPath: hconf.HypervisorCtlPath, + HypervisorCtlPathList: hconf.HypervisorCtlPathList, JailerPath: hconf.JailerPath, JailerPathList: hconf.JailerPathList, BlockDeviceDriver: hconf.BlockDeviceDriver, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 86b1f2565c..4b09e2aca2 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -66,6 +66,10 @@ type HypervisorConfig struct { // HypervisorCtlPath is the hypervisor ctl executable host path. HypervisorCtlPath string + // HypervisorCtlPathList is the list of hypervisor control paths names allowed in annotations + HypervisorCtlPathList []string + + // HypervisorCtlPath is the hypervisor ctl executable host path. // JailerPath is the jailer executable host path. JailerPath string diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index b9b3bf2fa3..14d50a3c0c 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -44,6 +44,9 @@ const ( // JailerPath is a sandbox annotation for passing a per container path pointing at the jailer that will constrain the container VM. JailerPath = kataAnnotHypervisorPrefix + "jailer_path" + // CtlPath is a sandbox annotation for passing a per container path pointing at the acrn ctl binary + CtlPath = kataAnnotHypervisorPrefix + "ctlpath" + // FirmwarePath is a sandbox annotation for passing a per container path pointing at the guest firmware that will run the container VM. FirmwarePath = kataAnnotHypervisorPrefix + "firmware" diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 236ea47dc4..c632a664d0 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -404,6 +404,13 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, config.HypervisorConfig.JailerPath = value } + if value, ok := ocispec.Annotations[vcAnnotations.CtlPath]; ok { + if !regexpContains(runtime.HypervisorConfig.HypervisorCtlPathList, value) { + return fmt.Errorf("hypervisor control %v required from annotation is not valid", value) + } + config.HypervisorConfig.HypervisorCtlPath = value + } + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { if value != "" { params := vc.DeserializeParams(strings.Fields(value)) From 8cd094cf06a94633bacd84bb214da4678147c042 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 18:26:29 +0200 Subject: [PATCH 077/124] config: Add security warning on configuration examples Add the following text explaining the risk of using regular expressions in path lists: Each member of the list can be a regular expression, but prefer names. Otherwise, please read and understand the following carefully. SECURITY WARNING: If you use regular expressions, be mindful that an attacker could craft an annotation that uses .. to escape the paths you gave. For example, if your regexp is /bin/qemu.* then if there is a directory named /bin/qemu.d/, then an attacker can pass an annotation containing /bin/qemu.d/../put-any-binary-name-here and attack your host. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/cli/config/configuration-acrn.toml.in | 8 +++++++- src/runtime/cli/config/configuration-clh.toml.in | 13 ++++++++++--- src/runtime/cli/config/configuration-fc.toml.in | 8 +++++++- .../config/configuration-qemu-virtiofs.toml.in | 8 +++++++- src/runtime/cli/config/configuration-qemu.toml.in | 15 ++++++++++++--- 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index f6e9b7849e..d56b2ec9bb 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -17,7 +17,13 @@ kernel = "@KERNELPATH_ACRN@" image = "@IMAGEPATH@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression +# Each member of the list can be a regular expression, but prefer names. +# Otherwise, please read and understand the following carefully. +# SECURITY WARNING: If you use regular expressions, be mindful that +# an attacker could craft an annotation that uses .. to escape the paths +# you gave. For example, if your regexp is /bin/qemu.* then if there is +# a directory named /bin/qemu.d/, then an attacker can pass an annotation +# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. # path_list = [ "@ACRNPATH@.*" ] # List of valid annotations values for ctlpath (default: empty) diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 2b9f1b6b58..8e2419f62e 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -12,12 +12,19 @@ [hypervisor.clh] path = "@CLHPATH@" -# List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression -# path_list = [ "@CLHPATH@.*" ] kernel = "@KERNELPATH_CLH@" image = "@IMAGEPATH@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression, but prefer names. +# Otherwise, please read and understand the following carefully. +# SECURITY WARNING: If you use regular expressions, be mindful that +# an attacker could craft an annotation that uses .. to escape the paths +# you gave. For example, if your regexp is /bin/qemu.* then if there is +# a directory named /bin/qemu.d/, then an attacker can pass an annotation +# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. +# path_list = [ "@CLHPATH@.*" ] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index f54b7ba4ae..2c239cd4e7 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -16,7 +16,13 @@ kernel = "@KERNELPATH_FC@" image = "@IMAGEPATH@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression +# Each member of the list can be a regular expression, but prefer names. +# Otherwise, please read and understand the following carefully. +# SECURITY WARNING: If you use regular expressions, be mindful that +# an attacker could craft an annotation that uses .. to escape the paths +# you gave. For example, if your regexp is /bin/qemu.* then if there is +# a directory named /bin/qemu.d/, then an attacker can pass an annotation +# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. # path_list = [ "@FCPATH@.*" ] # Path for the jailer specific to firecracker diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index cc0c63e483..ee27f6e62a 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -17,7 +17,13 @@ image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression +# Each member of the list can be a regular expression, but prefer names. +# Otherwise, please read and understand the following carefully. +# SECURITY WARNING: If you use regular expressions, be mindful that +# an attacker could craft an annotation that uses .. to escape the paths +# you gave. For example, if your regexp is /bin/qemu.* then if there is +# a directory named /bin/qemu.d/, then an attacker can pass an annotation +# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. # path_list = [ "@QEMUPATH@.*" ] # Optional space-separated list of options to pass to the guest kernel. diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 22054c0374..21c2a2b426 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -12,6 +12,15 @@ [hypervisor.qemu] path = "@QEMUPATH@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list can be a regular expression, but prefer names. +# Otherwise, please read and understand the following carefully. +# SECURITY WARNING: If you use regular expressions, be mindful that +# an attacker could craft an annotation that uses .. to escape the paths +# you gave. For example, if your regexp is /bin/qemu.* then if there is +# a directory named /bin/qemu.d/, then an attacker can pass an annotation +# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. +# path_list = [ "@QEMUPATH@.*" ] kernel = "@KERNELPATH@" image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" @@ -220,17 +229,17 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # This option changes the default hypervisor and kernel parameters # to enable debug output where available. -# +# # Default false #enable_debug = true # Disable the customizations done in the runtime when it detects # that it is running on top a VMM. This will result in the runtime # behaving as it would when running on bare metal. -# +# #disable_nesting_checks = true -# This is the msize used for 9p shares. It is the number of bytes +# This is the msize used for 9p shares. It is the number of bytes # used for 9p packet payload. #msize_9p = @DEFMSIZE9P@ From c2a186b18c91541c8917a3695dfde2b71af5baa0 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 18:42:30 +0200 Subject: [PATCH 078/124] config: Protect vhost_user_store_path against annotation attacks This path could be used to overwrite data on the host. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/cli/config/configuration-qemu-virtiofs.toml.in | 3 +++ src/runtime/cli/config/configuration-qemu.toml.in | 3 +++ src/runtime/pkg/katautils/config.go | 1 + src/runtime/virtcontainers/hypervisor.go | 3 +++ src/runtime/virtcontainers/persist.go | 2 ++ src/runtime/virtcontainers/persist/api/config.go | 3 +++ src/runtime/virtcontainers/pkg/oci/utils.go | 7 +++++++ 7 files changed, 22 insertions(+) diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index ee27f6e62a..6ec254fe55 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -212,6 +212,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true +# List of valid annotations values for the virtiofs daemon (default: empty) +# vhost_user_store_path_list = [ "/empty/space", "/multiverse/quantum-foam" ] + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 21c2a2b426..d1f63f7e7f 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -217,6 +217,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true +# List of valid annotations values for the virtiofs daemon (default: empty) +# vhost_user_store_path_list = [ "/empty/space", "/multiverse/quantum-foam" ] + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index adb398bd77..99d0166b06 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -680,6 +680,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableVhostNet: h.DisableVhostNet, EnableVhostUserStore: h.EnableVhostUserStore, VhostUserStorePath: h.vhostUserStorePath(), + VhostUserStorePathList: h.VhostUserStorePathList, GuestHookPath: h.guestHookPath(), RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index a50fd5f5ab..3ba2529351 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -412,6 +412,9 @@ type HypervisorConfig struct { // related folders, sockets and device nodes should be. VhostUserStorePath string + // VhostUserStorePathList is the list of valid values for vhost-user paths + VhostUserStorePathList []string + // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks GuestHookPath string diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 9e6789fe7b..e64c9b3b1d 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -247,6 +247,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { DisableVhostNet: sconfig.HypervisorConfig.DisableVhostNet, EnableVhostUserStore: sconfig.HypervisorConfig.EnableVhostUserStore, VhostUserStorePath: sconfig.HypervisorConfig.VhostUserStorePath, + VhostUserStorePathList: sconfig.HypervisorConfig.VhostUserStorePathList, GuestHookPath: sconfig.HypervisorConfig.GuestHookPath, VMid: sconfig.HypervisorConfig.VMid, RxRateLimiterMaxRate: sconfig.HypervisorConfig.RxRateLimiterMaxRate, @@ -512,6 +513,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { DisableVhostNet: hconf.DisableVhostNet, EnableVhostUserStore: hconf.EnableVhostUserStore, VhostUserStorePath: hconf.VhostUserStorePath, + VhostUserStorePathList: hconf.VhostUserStorePathList, GuestHookPath: hconf.GuestHookPath, VMid: hconf.VMid, RxRateLimiterMaxRate: hconf.RxRateLimiterMaxRate, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 4b09e2aca2..cdb58727c0 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -186,6 +186,9 @@ type HypervisorConfig struct { // related folders, sockets and device nodes should be. VhostUserStorePath string + // VhostUserStorePathList is the list of valid values for vhost-user paths + VhostUserStorePathList []string + // GuestHookPath is the path within the VM that will be used for 'drop-in' hooks GuestHookPath string diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index c632a664d0..dbff39dbc1 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -434,6 +434,13 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, } } + if value, ok := ocispec.Annotations[vcAnnotations.VhostUserStorePath]; ok { + if !regexpContains(runtime.HypervisorConfig.VhostUserStorePathList, value) { + return fmt.Errorf("vhost store path %v required from annotation is not valid", value) + } + config.HypervisorConfig.VhostUserStorePath = value + } + if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok { if value != "" { config.HypervisorConfig.GuestHookPath = value From 28c386c51f3b2f56d1d77436ab6925249caf1d1b Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 15 May 2020 18:55:02 +0200 Subject: [PATCH 079/124] config: Protect file_mem_backend against annotation attacks This one could theoretically be used to overwrite data on the host. It seems somewhat less risky than the earlier ones for a number of reasons, but worth protecting a little anyway. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/cli/config/configuration-qemu-virtiofs.toml.in | 3 +++ src/runtime/cli/config/configuration-qemu.toml.in | 3 +++ src/runtime/pkg/katautils/config.go | 2 ++ src/runtime/virtcontainers/hypervisor.go | 3 +++ src/runtime/virtcontainers/persist.go | 2 ++ src/runtime/virtcontainers/persist/api/config.go | 3 +++ src/runtime/virtcontainers/pkg/oci/utils.go | 7 +++++-- 7 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 6ec254fe55..d9564adfb3 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -221,6 +221,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # This option will be ignored if VM templating is enabled. #file_mem_backend = "" +# List of valid annotations values for the file_mem_backend annotation (default: empty) +# file_mem_backend_list = [ "/dev/shm" ] + # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true #enable_swap = true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index d1f63f7e7f..e87daa107e 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -226,6 +226,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # This option will be ignored if VM templating is enabled. #file_mem_backend = "" +# List of valid annotations values for the file_mem_backend annotation (default: empty) +# file_mem_backend_list = [ "/dev/shm" ] + # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true #enable_swap = true diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 99d0166b06..210a587472 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -665,6 +665,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { IOMMU: h.IOMMU, IOMMUPlatform: h.getIOMMUPlatform(), FileBackedMemRootDir: h.FileBackedMemRootDir, + FileBackedMemRootList: h.FileBackedMemRootList, Mlock: !h.Swap, Debug: h.Debug, DisableNestingChecks: h.DisableNestingChecks, @@ -824,6 +825,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { MemPrealloc: h.MemPrealloc, HugePages: h.HugePages, FileBackedMemRootDir: h.FileBackedMemRootDir, + FileBackedMemRootList: h.FileBackedMemRootList, Mlock: !h.Swap, Debug: h.Debug, DisableNestingChecks: h.DisableNestingChecks, diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 3ba2529351..e21b444b72 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -330,6 +330,9 @@ type HypervisorConfig struct { // File based memory backend root directory FileBackedMemRootDir string + // FileBackedMemRootList is the list of valid root directories values for annotations + FileBackedMemRootList []string + // customAssets is a map of assets. // Each value in that map takes precedence over the configured assets. // For example, if there is a value for the "kernel" key in this map, diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index e64c9b3b1d..50d525407e 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -236,6 +236,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { MemPrealloc: sconfig.HypervisorConfig.MemPrealloc, HugePages: sconfig.HypervisorConfig.HugePages, FileBackedMemRootDir: sconfig.HypervisorConfig.FileBackedMemRootDir, + FileBackedMemRootList: sconfig.HypervisorConfig.FileBackedMemRootList, Realtime: sconfig.HypervisorConfig.Realtime, Mlock: sconfig.HypervisorConfig.Mlock, DisableNestingChecks: sconfig.HypervisorConfig.DisableNestingChecks, @@ -502,6 +503,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { MemPrealloc: hconf.MemPrealloc, HugePages: hconf.HugePages, FileBackedMemRootDir: hconf.FileBackedMemRootDir, + FileBackedMemRootList: hconf.FileBackedMemRootList, Realtime: hconf.Realtime, Mlock: hconf.Mlock, DisableNestingChecks: hconf.DisableNestingChecks, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index cdb58727c0..5f6c2964b8 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -116,6 +116,9 @@ type HypervisorConfig struct { // File based memory backend root directory FileBackedMemRootDir string + // FileBackedMemRootList is the list of valid root directories values for annotations + FileBackedMemRootList []string + // BlockDeviceCacheSet specifies cache-related options will be set to block devices or not. BlockDeviceCacheSet bool diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index dbff39dbc1..6b13c00fd0 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -374,7 +374,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return err } - if err := addHypervisorMemoryOverrides(ocispec, config); err != nil { + if err := addHypervisorMemoryOverrides(ocispec, config, runtime); err != nil { return err } @@ -482,7 +482,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return nil } -func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { +func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { memorySz, err := strconv.ParseUint(value, 10, 32) if err != nil { @@ -546,6 +546,9 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig } if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok { + if !regexpContains(runtime.HypervisorConfig.FileBackedMemRootList, value) { + return fmt.Errorf("file_mem_backend value %v required from annotation is not valid", value) + } sbConfig.HypervisorConfig.FileBackedMemRootDir = value } From d71f9e1155082820a1a260acfea8c52d4fa5d631 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Tue, 19 May 2020 17:13:09 +0200 Subject: [PATCH 080/124] config: Add makefile variables for path lists Add variables to override defaults at build time for the various lists used to control path annotations. Fixes: #901 Suggested-by: Fabiano Fidencio Signed-off-by: Christophe de Dinechin --- src/runtime/Makefile | 33 +++++++++++++++++++ .../cli/config/configuration-acrn.toml.in | 12 ++----- .../cli/config/configuration-clh.toml.in | 12 ++----- .../cli/config/configuration-fc.toml.in | 12 ++----- .../configuration-qemu-virtiofs.toml.in | 18 ++++------ .../cli/config/configuration-qemu.toml.in | 23 +++++-------- 6 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 2a6730d4ef..e8bb5fe9ca 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -129,16 +129,22 @@ DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU) HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_CLH) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) +QEMUPATHLIST := [\"$(QEMUPATH)\"] QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) CLHPATH := $(CLHBINDIR)/$(CLHCMD) +CLHPATHLIST := [\"$(CLHBINDIR)/$(CLHCMD)\"] FCPATH = $(FCBINDIR)/$(FCCMD) +FCPATHLIST = [\"$(FCPATH)\"] FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) +FCJAILERPATHLIST = [\"$(FCJAILERPATH)\"] ACRNPATH := $(ACRNBINDIR)/$(ACRNCMD) +ACRNPATHLIST := [\"$(ACRNPATH)\"] ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) +ACRNCTLPATHLIST := [\"$(ACRNCTLPATH)\"] SHIMCMD := $(BIN_PREFIX)-shim SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) @@ -172,6 +178,7 @@ DEFDISABLEBLOCK := false DEFSHAREDFS := virtio-9p DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd +DEFVIRTIOFSDAEMONLIST := [\"$(DEFVIRTIOFSDAEMON)\"] # Default DAX mapping cache size in MiB #if value is 0, DAX is not enabled DEFVIRTIOFSCACHESIZE := 0 @@ -187,6 +194,9 @@ DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false DEFENABLEVHOSTUSERSTORE := false DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user +DEFVHOSTUSERSTOREPATHLIST := [\"$(DEFVHOSTUSERSTOREPATH)\"] +DEFFILEMEMBACKEND := "" +DEFFILEMEMBACKENDLIST := [\"$(DEFFILEMEMBACKEND)\"] DEFENABLESWAP := false DEFENABLEDEBUG := false DEFDISABLENESTINGCHECKS := false @@ -391,10 +401,16 @@ USER_VARS += DEFAULT_HYPERVISOR USER_VARS += ACRNCMD USER_VARS += ACRNCTLCMD USER_VARS += ACRNPATH +USER_VARS += ACRNPATHLIST USER_VARS += ACRNCTLPATH +USER_VARS += ACRNCTLPATHLIST +USER_VARS += CLHPATH +USER_VARS += CLHPATHLIST USER_VARS += FCCMD USER_VARS += FCPATH +USER_VARS += FCPATHLIST USER_VARS += FCJAILERPATH +USER_VARS += FCJAILERPATHLIST USER_VARS += SYSCONFIG USER_VARS += IMAGENAME USER_VARS += IMAGEPATH @@ -425,8 +441,10 @@ USER_VARS += NETMONPATH USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH +USER_VARS += QEMUPATHLIST USER_VARS += QEMUVIRTIOFSCMD USER_VARS += QEMUVIRTIOFSPATH +USER_VARS += QEMUVIRTIOFSPATHLIST USER_VARS += SHAREDIR USER_VARS += SHIMPATH USER_VARS += SYSCONFDIR @@ -449,6 +467,7 @@ USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS USER_VARS += DEFSHAREDFS USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON +USER_VARS += DEFVIRTIOFSDAEMONLIST USER_VARS += DEFVIRTIOFSCACHESIZE USER_VARS += DEFVIRTIOFSCACHE USER_VARS += DEFVIRTIOFSEXTRAARGS @@ -457,6 +476,9 @@ USER_VARS += DEFENABLEMEMPREALLOC USER_VARS += DEFENABLEHUGEPAGES USER_VARS += DEFENABLEVHOSTUSERSTORE USER_VARS += DEFVHOSTUSERSTOREPATH +USER_VARS += DEFVHOSTUSERSTOREPATHLIST +USER_VARS += DEFFILEMEMBACKEND +USER_VARS += DEFFILEMEMBACKENDLIST USER_VARS += DEFENABLESWAP USER_VARS += DEFENABLEDEBUG USER_VARS += DEFDISABLENESTINGCHECKS @@ -606,10 +628,15 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ + -e "s|@FCPATHLIST@|$(FCPATHLIST)|g" \ -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ + -e "s|@FCJAILERPATHLIST@|$(FCJAILERPATHLIST)|g" \ -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ + -e "s|@ACRNPATHLIST@|$(ACRNPATHLIST)|g" \ -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ + -e "s|@ACRNCTLPATHLIST@|$(ACRNCTLPATHLIST)|g" \ -e "s|@CLHPATH@|$(CLHPATH)|g" \ + -e "s|@CLHPATHLIST@|$(CLHPATHLIST)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ @@ -635,7 +662,9 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@PROJECT_TAG@|$(PROJECT_TAG)|g" \ -e "s|@PROJECT_TYPE@|$(PROJECT_TYPE)|g" \ -e "s|@QEMUPATH@|$(QEMUPATH)|g" \ + -e "s|@QEMUPATHLIST@|$(QEMUPATHLIST)|g" \ -e "s|@QEMUVIRTIOFSPATH@|$(QEMUVIRTIOFSPATH)|g" \ + -e "s|@QEMUVIRTIOFSPATHLIST@|$(QEMUVIRTIOFSPATHLIST)|g" \ -e "s|@RUNTIME_NAME@|$(TARGET)|g" \ -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ @@ -659,6 +688,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \ -e "s|@DEFSHAREDFS_QEMU_VIRTIOFS@|$(DEFSHAREDFS_QEMU_VIRTIOFS)|g" \ -e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \ + -e "s|@DEFVIRTIOFSDAEMONLIST@|$(DEFVIRTIOFSDAEMONLIST)|g" \ -e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \ -e "s|@DEFVIRTIOFSCACHE@|$(DEFVIRTIOFSCACHE)|g" \ -e "s|@DEFVIRTIOFSEXTRAARGS@|$(DEFVIRTIOFSEXTRAARGS)|g" \ @@ -667,6 +697,9 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \ -e "s|@DEFENABLEVHOSTUSERSTORE@|$(DEFENABLEVHOSTUSERSTORE)|g" \ -e "s|@DEFVHOSTUSERSTOREPATH@|$(DEFVHOSTUSERSTOREPATH)|g" \ + -e "s|@DEFVHOSTUSERSTOREPATHLIST@|$(DEFVHOSTUSERSTOREPATHLIST)|g" \ + -e "s|@DEFFILEMEMBACKEND@|$(DEFFILEMEMBACKEND)|g" \ + -e "s|@DEFFILEMEMBACKENDLIST@|$(DEFFILEMEMBACKENDLIST)|g" \ -e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \ -e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \ -e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \ diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index d56b2ec9bb..7478fed93d 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -17,17 +17,11 @@ kernel = "@KERNELPATH_ACRN@" image = "@IMAGEPATH@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression, but prefer names. -# Otherwise, please read and understand the following carefully. -# SECURITY WARNING: If you use regular expressions, be mindful that -# an attacker could craft an annotation that uses .. to escape the paths -# you gave. For example, if your regexp is /bin/qemu.* then if there is -# a directory named /bin/qemu.d/, then an attacker can pass an annotation -# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. -# path_list = [ "@ACRNPATH@.*" ] +# Each member of the list is a path pattern as described by glob(3). +path_list = @ACRNPATHLIST@ # List of valid annotations values for ctlpath (default: empty) -# ctlpath_list = [ "@ACRNCTLPATH@.*" ] +ctlpath_list = @ACRNCTLPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 8e2419f62e..f1f02a129b 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -16,14 +16,8 @@ kernel = "@KERNELPATH_CLH@" image = "@IMAGEPATH@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression, but prefer names. -# Otherwise, please read and understand the following carefully. -# SECURITY WARNING: If you use regular expressions, be mindful that -# an attacker could craft an annotation that uses .. to escape the paths -# you gave. For example, if your regexp is /bin/qemu.* then if there is -# a directory named /bin/qemu.d/, then an attacker can pass an annotation -# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. -# path_list = [ "@CLHPATH@.*" ] +# Each member of the list is a path pattern as described by glob(3). +path_list = @CLHPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -73,7 +67,7 @@ default_memory = @DEFMEMSZ@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon (default: empty) -# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] +virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 2c239cd4e7..b6fd8647d5 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -16,14 +16,8 @@ kernel = "@KERNELPATH_FC@" image = "@IMAGEPATH@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression, but prefer names. -# Otherwise, please read and understand the following carefully. -# SECURITY WARNING: If you use regular expressions, be mindful that -# an attacker could craft an annotation that uses .. to escape the paths -# you gave. For example, if your regexp is /bin/qemu.* then if there is -# a directory named /bin/qemu.d/, then an attacker can pass an annotation -# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. -# path_list = [ "@FCPATH@.*" ] +# Each member of the list is a path pattern as described by glob(3). +path_list = @FCPATHLIST@ # Path for the jailer specific to firecracker # If the jailer path is not set kata will launch firecracker @@ -35,7 +29,7 @@ image = "@IMAGEPATH@" # List of valid jailer path values for the hypervisor (default: empty) # Each member of the list can be a regular expression -# jailer_path_list = [ "@FCJAILERPATH@.*" ] +# jailer_path_list = @FCJAILERPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index d9564adfb3..22ec8d91e1 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -17,14 +17,8 @@ image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" # List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression, but prefer names. -# Otherwise, please read and understand the following carefully. -# SECURITY WARNING: If you use regular expressions, be mindful that -# an attacker could craft an annotation that uses .. to escape the paths -# you gave. For example, if your regexp is /bin/qemu.* then if there is -# a directory named /bin/qemu.d/, then an attacker can pass an annotation -# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. -# path_list = [ "@QEMUPATH@.*" ] +# Each member of the list is a path pattern as described by glob(3). +path_list = @QEMUVIRTIOFSPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -121,7 +115,7 @@ shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon (default: empty) -# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] +virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -213,16 +207,16 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" #enable_iommu_platform = true # List of valid annotations values for the virtiofs daemon (default: empty) -# vhost_user_store_path_list = [ "/empty/space", "/multiverse/quantum-foam" ] +vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. # This option will be ignored if VM templating is enabled. -#file_mem_backend = "" +#file_mem_backend = "@DEFFILEMEMBACKEND@" # List of valid annotations values for the file_mem_backend annotation (default: empty) -# file_mem_backend_list = [ "/dev/shm" ] +#file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index e87daa107e..9891d2002c 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -12,19 +12,14 @@ [hypervisor.qemu] path = "@QEMUPATH@" -# List of valid annotations values for the hypervisor (default: empty) -# Each member of the list can be a regular expression, but prefer names. -# Otherwise, please read and understand the following carefully. -# SECURITY WARNING: If you use regular expressions, be mindful that -# an attacker could craft an annotation that uses .. to escape the paths -# you gave. For example, if your regexp is /bin/qemu.* then if there is -# a directory named /bin/qemu.d/, then an attacker can pass an annotation -# containing /bin/qemu.d/../put-any-binary-name-here and attack your host. -# path_list = [ "@QEMUPATH@.*" ] kernel = "@KERNELPATH@" image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" +# List of valid annotations values for the hypervisor (default: empty) +# Each member of the list is a path pattern as described by glob(3). +path_list = @QEMUPATHLIST@ + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. @@ -126,7 +121,7 @@ shared_fs = "@DEFSHAREDFS@" virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon (default: empty) -# virtio_fs_daemon_list = [ "/opt/kata/bin/virtiofsd", "/usr/.*/virtiofsd" ] +virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -217,17 +212,17 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true -# List of valid annotations values for the virtiofs daemon (default: empty) -# vhost_user_store_path_list = [ "/empty/space", "/multiverse/quantum-foam" ] +# List of valid annotations values for the vhost user store path (default: empty) +vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. # This option will be ignored if VM templating is enabled. -#file_mem_backend = "" +#file_mem_backend = "@DEFFILEMEMBACKEND@" # List of valid annotations values for the file_mem_backend annotation (default: empty) -# file_mem_backend_list = [ "/dev/shm" ] +#file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true From 29f5dec38fdffca072873d46968c583b18604d91 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Tue, 19 May 2020 17:52:32 +0200 Subject: [PATCH 081/124] annotations: Fix typo in comment A comment talking about runtime related annotations describes them as being related to the agent. A similar comment for the agent annotations is missing. Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/annotations/annotations.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 14d50a3c0c..44379b5d1e 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -214,7 +214,7 @@ const ( TxRateLimiterMaxRate = kataAnnotHypervisorPrefix + "tx_rate_limiter_max_rate" ) -// Agent related annotations +// Runtime related annotations const ( kataAnnotRuntimePrefix = kataConfAnnotationsPrefix + "runtime." @@ -238,6 +238,7 @@ const ( DisableNewNetNs = kataAnnotRuntimePrefix + "disable_new_netns" ) +// Agent related annotations const ( kataAnnotAgentPrefix = kataConfAnnotationsPrefix + "agent." From 5a587ba5069a89e972f6c29a837ddb67b88d6d87 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Mon, 18 May 2020 19:00:54 +0200 Subject: [PATCH 082/124] config: Use glob instead of regexp to match paths in annotations When filtering annotations that correspond to paths, e.g. hypervisor.path, it is better to use a glob syntax than a regexp syntax, as it is more usual for paths, and prevents classes of matches that are undesirable in our case, such as matching .. against .* Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 24 +++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 6b13c00fd0..bffa931661 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -200,6 +200,18 @@ func regexpContains(s []string, e string) bool { return false } +func checkPathIsInGlobList(list []string, path string) bool { + for _, glob := range list { + filenames, _ := filepath.Glob(glob) + for _, a := range filenames { + if path == a { + return true + } + } + } + return false +} + func newLinuxDeviceInfo(d specs.LinuxDevice) (*config.DeviceInfo, error) { allowedDeviceTypes := []string{"c", "b", "u", "p"} @@ -391,21 +403,21 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, } if value, ok := ocispec.Annotations[vcAnnotations.HypervisorPath]; ok { - if !regexpContains(runtime.HypervisorConfig.HypervisorPathList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.HypervisorPathList, value) { return fmt.Errorf("hypervisor %v required from annotation is not valid", value) } config.HypervisorConfig.HypervisorPath = value } if value, ok := ocispec.Annotations[vcAnnotations.JailerPath]; ok { - if !regexpContains(runtime.HypervisorConfig.JailerPathList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.JailerPathList, value) { return fmt.Errorf("jailer %v required from annotation is not valid", value) } config.HypervisorConfig.JailerPath = value } if value, ok := ocispec.Annotations[vcAnnotations.CtlPath]; ok { - if !regexpContains(runtime.HypervisorConfig.HypervisorCtlPathList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.HypervisorCtlPathList, value) { return fmt.Errorf("hypervisor control %v required from annotation is not valid", value) } config.HypervisorConfig.HypervisorCtlPath = value @@ -435,7 +447,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, } if value, ok := ocispec.Annotations[vcAnnotations.VhostUserStorePath]; ok { - if !regexpContains(runtime.HypervisorConfig.VhostUserStorePathList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.VhostUserStorePathList, value) { return fmt.Errorf("vhost store path %v required from annotation is not valid", value) } config.HypervisorConfig.VhostUserStorePath = value @@ -546,7 +558,7 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig } if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok { - if !regexpContains(runtime.HypervisorConfig.FileBackedMemRootList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.FileBackedMemRootList, value) { return fmt.Errorf("file_mem_backend value %v required from annotation is not valid", value) } sbConfig.HypervisorConfig.FileBackedMemRootDir = value @@ -702,7 +714,7 @@ func addHypervisorVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConf } if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok { - if !regexpContains(runtime.HypervisorConfig.VirtioFSDaemonList, value) { + if !checkPathIsInGlobList(runtime.HypervisorConfig.VirtioFSDaemonList, value) { return fmt.Errorf("virtiofs daemon %v required from annotation is not valid", value) } sbConfig.HypervisorConfig.VirtioFSDaemon = value From 73bb3fdbee220bea6d1baab7c3e77e70a4e6f60d Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Tue, 19 May 2020 19:19:42 +0200 Subject: [PATCH 083/124] config: Whitelist hypervisor annotations by name Add a field "enable_annotations" to the runtime configuration that can be used to whitelist annotations using a list of regular expressions, which are used to match any part of the base annotation name, i.e. the part after "io.katacontainers.config.hypervisor." For example, the following configuraiton will match "virtio_fs_daemon", "initrd" and "jailer_path", but not "path" nor "firmware": enable_annotations = [ "virtio.*", "initrd", "_path" ] The default is an empty list of enabled annotations, which disables annotations entirely. If an anontation is rejected, the message is something like: annotation io.katacontainers.config.hypervisor.virtio_fs_daemon is not enabled Fixes: #901 Suggested-by: Peng Tao Signed-off-by: Christophe de Dinechin --- src/runtime/Makefile | 2 ++ src/runtime/cli/config/configuration-acrn.toml.in | 5 +++++ src/runtime/cli/config/configuration-clh.toml.in | 5 +++++ src/runtime/cli/config/configuration-fc.toml.in | 5 +++++ .../cli/config/configuration-qemu-virtiofs.toml.in | 5 +++++ src/runtime/cli/config/configuration-qemu.toml.in | 5 +++++ src/runtime/pkg/katautils/config.go | 5 +++++ src/runtime/virtcontainers/hypervisor.go | 3 +++ src/runtime/virtcontainers/persist.go | 2 ++ src/runtime/virtcontainers/persist/api/config.go | 3 +++ .../virtcontainers/pkg/annotations/annotations.go | 1 + src/runtime/virtcontainers/pkg/oci/utils.go | 13 +++++++++++++ 12 files changed, 54 insertions(+) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index e8bb5fe9ca..784c4c705c 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -167,6 +167,7 @@ DEFMEMSZ := 2048 DEFMEMSLOTS := 10 #Default number of bridges DEFBRIDGES := 1 +DEFENABLEANNOTATIONS := [] DEFDISABLEGUESTSECCOMP := true #Default experimental features enabled DEFAULTEXPFEATURES := [] @@ -678,6 +679,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFNETWORKMODEL_CLH@|$(DEFNETWORKMODEL_CLH)|g" \ -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ + -e "s|@DEFENABLEANNOTATIONS@|$(DEFENABLEANNOTATIONS)|g" \ -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ -e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \ -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 7478fed93d..3077495277 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -16,6 +16,11 @@ ctlpath = "@ACRNCTLPATH@" kernel = "@KERNELPATH_ACRN@" image = "@IMAGEPATH@" +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + # List of valid annotations values for the hypervisor (default: empty) # Each member of the list is a path pattern as described by glob(3). path_list = @ACRNPATHLIST@ diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index f1f02a129b..0dde89c851 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -15,6 +15,11 @@ path = "@CLHPATH@" kernel = "@KERNELPATH_CLH@" image = "@IMAGEPATH@" +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + # List of valid annotations values for the hypervisor (default: empty) # Each member of the list is a path pattern as described by glob(3). path_list = @CLHPATHLIST@ diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index b6fd8647d5..403928886f 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -15,6 +15,11 @@ path = "@FCPATH@" kernel = "@KERNELPATH_FC@" image = "@IMAGEPATH@" +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + # List of valid annotations values for the hypervisor (default: empty) # Each member of the list is a path pattern as described by glob(3). path_list = @FCPATHLIST@ diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 22ec8d91e1..afde7e5f90 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -16,6 +16,11 @@ kernel = "@KERNELVIRTIOFSPATH@" image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + # List of valid annotations values for the hypervisor (default: empty) # Each member of the list is a path pattern as described by glob(3). path_list = @QEMUVIRTIOFSPATHLIST@ diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 9891d2002c..cb52589377 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -16,6 +16,11 @@ kernel = "@KERNELPATH@" image = "@IMAGEPATH@" machine_type = "@MACHINETYPE@" +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = @DEFENABLEANNOTATIONS@ + # List of valid annotations values for the hypervisor (default: empty) # Each member of the list is a path pattern as described by glob(3). path_list = @QEMUPATHLIST@ diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 210a587472..ba9f841dee 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -124,6 +124,7 @@ type hypervisor struct { GuestHookPath string `toml:"guest_hook_path"` RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` + EnableAnnotations []string `toml:"enable_annotations"` } type runtime struct { @@ -558,6 +559,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { GuestHookPath: h.guestHookPath(), RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, + EnableAnnotations: h.EnableAnnotations, }, nil } @@ -685,6 +687,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { GuestHookPath: h.guestHookPath(), RxRateLimiterMaxRate: rxRateLimiterMaxRate, TxRateLimiterMaxRate: txRateLimiterMaxRate, + EnableAnnotations: h.EnableAnnotations, }, nil } @@ -748,6 +751,7 @@ func newAcrnHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { BlockDeviceDriver: blockDriver, DisableVhostNet: h.DisableVhostNet, GuestHookPath: h.guestHookPath(), + EnableAnnotations: h.EnableAnnotations, }, nil } @@ -839,6 +843,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { PCIeRootPort: h.PCIeRootPort, DisableVhostNet: true, VirtioFSExtraArgs: h.VirtioFSExtraArgs, + EnableAnnotations: h.EnableAnnotations, }, nil } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index e21b444b72..95ad7d609d 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -433,6 +433,9 @@ type HypervisorConfig struct { // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. TxRateLimiterMaxRate uint64 + + // Enable annotations by name + EnableAnnotations []string } // vcpu mapping from vcpu number to thread number diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 50d525407e..49c0f49d6f 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -253,6 +253,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { VMid: sconfig.HypervisorConfig.VMid, RxRateLimiterMaxRate: sconfig.HypervisorConfig.RxRateLimiterMaxRate, TxRateLimiterMaxRate: sconfig.HypervisorConfig.TxRateLimiterMaxRate, + EnableAnnotations: sconfig.HypervisorConfig.EnableAnnotations, } ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{ @@ -520,6 +521,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { VMid: hconf.VMid, RxRateLimiterMaxRate: hconf.RxRateLimiterMaxRate, TxRateLimiterMaxRate: hconf.TxRateLimiterMaxRate, + EnableAnnotations: hconf.EnableAnnotations, } sconfig.AgentConfig = KataAgentConfig{ diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 5f6c2964b8..86f6a66e1e 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -204,6 +204,9 @@ type HypervisorConfig struct { // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. TxRateLimiterMaxRate uint64 + + // Enable annotations by name + EnableAnnotations []string } // KataAgentConfig is a structure storing information needed diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 44379b5d1e..c53a152460 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -28,6 +28,7 @@ const ( // // Assets // + KataAnnotationHypervisorPrefix = kataAnnotHypervisorPrefix // KernelPath is a sandbox annotation for passing a per container path pointing at the kernel needed to boot the container VM. KernelPath = kataAnnotHypervisorPrefix + "kernel" diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index bffa931661..d6ba8f751e 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -212,6 +212,14 @@ func checkPathIsInGlobList(list []string, path string) bool { return false } +// Check if an annotation name either belongs to another prefix, matches regexp list +func checkAnnotationNameIsValid(list []string, name string, prefix string) bool { + if strings.HasPrefix(name, prefix) { + return regexpContains(list, strings.TrimPrefix(name, prefix)) + } + return true +} + func newLinuxDeviceInfo(d specs.LinuxDevice) (*config.DeviceInfo, error) { allowedDeviceTypes := []string{"c", "b", "u", "p"} @@ -345,6 +353,11 @@ func SandboxID(spec specs.Spec) (string, error) { } func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig, runtime RuntimeConfig) error { + for key := range ocispec.Annotations { + if !checkAnnotationNameIsValid(runtime.HypervisorConfig.EnableAnnotations, key, vcAnnotations.KataAnnotationHypervisorPrefix) { + return fmt.Errorf("annotation %v is not enabled", key) + } + } addAssetAnnotations(ocispec, config) if err := addHypervisorConfigOverrides(ocispec, config, runtime); err != nil { return err From 74d4065197ac5b597c9ebfee3bf6a0a20c0981be Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 3 Sep 2020 13:42:12 +0200 Subject: [PATCH 084/124] config: Add better comments in the template files When there is a default value from the code (usually empty) that differs from a possible suggested value from the distro, then the wording "default: empty" is confusing. Fixes: #901 Suggested-by: Julio Montes Signed-off-by: Christophe de Dinechin --- .../cli/config/configuration-acrn.toml.in | 8 ++++++-- .../cli/config/configuration-clh.toml.in | 8 ++++++-- .../cli/config/configuration-fc.toml.in | 10 +++++++--- .../configuration-qemu-virtiofs.toml.in | 20 ++++++++++++++----- .../cli/config/configuration-qemu.toml.in | 18 ++++++++++++----- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 3077495277..85d60cd443 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -21,11 +21,15 @@ image = "@IMAGEPATH@" # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" enable_annotations = @DEFENABLEANNOTATIONS@ -# List of valid annotations values for the hypervisor (default: empty) +# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @ACRNPATHLIST@ path_list = @ACRNPATHLIST@ -# List of valid annotations values for ctlpath (default: empty) +# List of valid annotations values for ctlpath +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @ACRNCTLPATHLIST@ ctlpath_list = @ACRNCTLPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 0dde89c851..33379c8852 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -20,8 +20,10 @@ image = "@IMAGEPATH@" # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" enable_annotations = @DEFENABLEANNOTATIONS@ -# List of valid annotations values for the hypervisor (default: empty) +# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @CLHPATHLIST@ path_list = @CLHPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. @@ -71,7 +73,9 @@ default_memory = @DEFMEMSZ@ # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" -# List of valid annotations values for the virtiofs daemon (default: empty) +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 403928886f..e21d2c1ddc 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -20,8 +20,10 @@ image = "@IMAGEPATH@" # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" enable_annotations = @DEFENABLEANNOTATIONS@ -# List of valid annotations values for the hypervisor (default: empty) +# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @FCPATHLIST@ path_list = @FCPATHLIST@ # Path for the jailer specific to firecracker @@ -32,9 +34,11 @@ path_list = @FCPATHLIST@ # for this feature today. #jailer_path = "@FCJAILERPATH@" -# List of valid jailer path values for the hypervisor (default: empty) +# List of valid jailer path values for the hypervisor # Each member of the list can be a regular expression -# jailer_path_list = @FCJAILERPATHLIST@ +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @FCJAILERPATHLIST@ +jailer_path_list = @FCJAILERPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index afde7e5f90..27c963d749 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -19,10 +19,14 @@ machine_type = "@MACHINETYPE@" # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFENABLEANNOTATIONS@ enable_annotations = @DEFENABLEANNOTATIONS@ -# List of valid annotations values for the hypervisor (default: empty) +# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @QEMUVIRTIOFSPATHLIST@ path_list = @QEMUVIRTIOFSPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. @@ -119,7 +123,9 @@ shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" -# List of valid annotations values for the virtiofs daemon (default: empty) +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB @@ -211,7 +217,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true -# List of valid annotations values for the virtiofs daemon (default: empty) +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVHOSTUSERSTOREPATHLIST@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # Enable file based guest memory support. The default is an empty string which @@ -220,8 +228,10 @@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # This option will be ignored if VM templating is enabled. #file_mem_backend = "@DEFFILEMEMBACKEND@" -# List of valid annotations values for the file_mem_backend annotation (default: empty) -#file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFFILEMEMBACKENDLIST@ +file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index cb52589377..87b10d3175 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -21,8 +21,10 @@ machine_type = "@MACHINETYPE@" # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" enable_annotations = @DEFENABLEANNOTATIONS@ -# List of valid annotations values for the hypervisor (default: empty) +# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @QEMUPATHLIST@ path_list = @QEMUPATHLIST@ # Optional space-separated list of options to pass to the guest kernel. @@ -125,7 +127,9 @@ shared_fs = "@DEFSHAREDFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" -# List of valid annotations values for the virtiofs daemon (default: empty) +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ # Default size of DAX cache in MiB @@ -217,7 +221,9 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # Enabling this will result in the VM device having iommu_platform=on set #enable_iommu_platform = true -# List of valid annotations values for the vhost user store path (default: empty) +# List of valid annotations values for the vhost user store path +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFVHOSTUSERSTOREPATHLIST@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # Enable file based guest memory support. The default is an empty string which @@ -226,8 +232,10 @@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # This option will be ignored if VM templating is enabled. #file_mem_backend = "@DEFFILEMEMBACKEND@" -# List of valid annotations values for the file_mem_backend annotation (default: empty) -#file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: @DEFFILEMEMBACKENDLIST@ +file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true From 997f7c4433a9f4325e7a126d58845bc48d7f6e68 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 10 Sep 2020 19:47:41 +0200 Subject: [PATCH 085/124] annotations: Rename checkPathIsInGlobList with checkPathIsInGlobs The name is shorter and more specific Fixes: #901 Suggested-by: James O.D. Hunt Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index d6ba8f751e..f030360856 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -200,8 +200,8 @@ func regexpContains(s []string, e string) bool { return false } -func checkPathIsInGlobList(list []string, path string) bool { - for _, glob := range list { +func checkPathIsInGlobs(globs []string, path string) bool { + for _, glob := range globs { filenames, _ := filepath.Glob(glob) for _, a := range filenames { if path == a { @@ -416,21 +416,21 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, } if value, ok := ocispec.Annotations[vcAnnotations.HypervisorPath]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.HypervisorPathList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorPathList, value) { return fmt.Errorf("hypervisor %v required from annotation is not valid", value) } config.HypervisorConfig.HypervisorPath = value } if value, ok := ocispec.Annotations[vcAnnotations.JailerPath]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.JailerPathList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.JailerPathList, value) { return fmt.Errorf("jailer %v required from annotation is not valid", value) } config.HypervisorConfig.JailerPath = value } if value, ok := ocispec.Annotations[vcAnnotations.CtlPath]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.HypervisorCtlPathList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorCtlPathList, value) { return fmt.Errorf("hypervisor control %v required from annotation is not valid", value) } config.HypervisorConfig.HypervisorCtlPath = value @@ -460,7 +460,7 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, } if value, ok := ocispec.Annotations[vcAnnotations.VhostUserStorePath]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.VhostUserStorePathList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.VhostUserStorePathList, value) { return fmt.Errorf("vhost store path %v required from annotation is not valid", value) } config.HypervisorConfig.VhostUserStorePath = value @@ -571,7 +571,7 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig } if value, ok := ocispec.Annotations[vcAnnotations.FileBackedMemRootDir]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.FileBackedMemRootList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.FileBackedMemRootList, value) { return fmt.Errorf("file_mem_backend value %v required from annotation is not valid", value) } sbConfig.HypervisorConfig.FileBackedMemRootDir = value @@ -727,7 +727,7 @@ func addHypervisorVirtioFsOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConf } if value, ok := ocispec.Annotations[vcAnnotations.VirtioFSDaemon]; ok { - if !checkPathIsInGlobList(runtime.HypervisorConfig.VirtioFSDaemonList, value) { + if !checkPathIsInGlobs(runtime.HypervisorConfig.VirtioFSDaemonList, value) { return fmt.Errorf("virtiofs daemon %v required from annotation is not valid", value) } sbConfig.HypervisorConfig.VirtioFSDaemon = value From a92a63031d4b00a4da8f61dabb4704dc9c306286 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 10 Sep 2020 19:54:12 +0200 Subject: [PATCH 086/124] annotations: Give better names to local variabes in search functions Use more meaningful variable names for clarity. Fixes: #901 Suggested-by: James O.D. Hunt james.o.hunt@intel.com> Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index f030360856..1dd9942c0c 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -182,18 +182,18 @@ func containerMounts(spec specs.Spec) []vc.Mount { return mnts } -func contains(s []string, e string) bool { - for _, a := range s { - if a == e { +func contains(strings []string, toFind string) bool { + for _, candidate := range strings { + if candidate == toFind { return true } } return false } -func regexpContains(s []string, e string) bool { - for _, a := range s { - if matched, _ := regexp.MatchString(a, e); matched { +func regexpContains(regexps []string, toMatch string) bool { + for _, candidate := range regexps { + if matched, _ := regexp.MatchString(candidate, toMatch); matched { return true } } From 83281365750829b68f3103a26860fd9e3ce1add7 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 10 Sep 2020 21:22:59 +0200 Subject: [PATCH 087/124] makefile: Improve names of config entries for annotation checks The entries used to be things like PATH_LIST, which are too generic. Replace them with more precise name with a distinguishing keyword, namely VALID. For example valid_hypervisor_paths. Fixes: #901 Suggested-by: James O.D. Hunt Signed-off-by: Christophe de Dinechin --- src/runtime/Makefile | 58 +++++++++---------- .../cli/config/configuration-acrn.toml.in | 8 +-- .../cli/config/configuration-clh.toml.in | 8 +-- .../cli/config/configuration-fc.toml.in | 8 +-- .../configuration-qemu-virtiofs.toml.in | 16 ++--- .../cli/config/configuration-qemu.toml.in | 16 ++--- src/runtime/pkg/katautils/config.go | 12 ++-- 7 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 784c4c705c..8d657e7643 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -129,22 +129,22 @@ DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU) HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_CLH) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) -QEMUPATHLIST := [\"$(QEMUPATH)\"] +QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"] -QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) +QEMUVALIDVIRTIOFSPATHS := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) CLHPATH := $(CLHBINDIR)/$(CLHCMD) -CLHPATHLIST := [\"$(CLHBINDIR)/$(CLHCMD)\"] +CLHVALIDHYPERVISORPATHS := [\"$(CLHBINDIR)/$(CLHCMD)\"] FCPATH = $(FCBINDIR)/$(FCCMD) -FCPATHLIST = [\"$(FCPATH)\"] +FCVALIDPATHS = [\"$(FCPATH)\"] FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) -FCJAILERPATHLIST = [\"$(FCJAILERPATH)\"] +FCVALIDJAILERPATHS = [\"$(FCJAILERPATH)\"] ACRNPATH := $(ACRNBINDIR)/$(ACRNCMD) -ACRNPATHLIST := [\"$(ACRNPATH)\"] +ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"] ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) -ACRNCTLPATHLIST := [\"$(ACRNCTLPATH)\"] +ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] SHIMCMD := $(BIN_PREFIX)-shim SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) @@ -179,7 +179,7 @@ DEFDISABLEBLOCK := false DEFSHAREDFS := virtio-9p DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd -DEFVIRTIOFSDAEMONLIST := [\"$(DEFVIRTIOFSDAEMON)\"] +DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"] # Default DAX mapping cache size in MiB #if value is 0, DAX is not enabled DEFVIRTIOFSCACHESIZE := 0 @@ -195,9 +195,9 @@ DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false DEFENABLEVHOSTUSERSTORE := false DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user -DEFVHOSTUSERSTOREPATHLIST := [\"$(DEFVHOSTUSERSTOREPATH)\"] +DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"] DEFFILEMEMBACKEND := "" -DEFFILEMEMBACKENDLIST := [\"$(DEFFILEMEMBACKEND)\"] +DEFVALIDFILEMEMBACKENDS := [\"$(DEFFILEMEMBACKEND)\"] DEFENABLESWAP := false DEFENABLEDEBUG := false DEFDISABLENESTINGCHECKS := false @@ -404,14 +404,14 @@ USER_VARS += ACRNCTLCMD USER_VARS += ACRNPATH USER_VARS += ACRNPATHLIST USER_VARS += ACRNCTLPATH -USER_VARS += ACRNCTLPATHLIST +USER_VARS += ACRNVALIDCTLPATHS USER_VARS += CLHPATH -USER_VARS += CLHPATHLIST +USER_VARS += CLHVALIDHYPERVISORPATHS USER_VARS += FCCMD USER_VARS += FCPATH -USER_VARS += FCPATHLIST +USER_VARS += FCVALIDHYPERVISORPATHS USER_VARS += FCJAILERPATH -USER_VARS += FCJAILERPATHLIST +USER_VARS += FCVALIDJAILERPATHS USER_VARS += SYSCONFIG USER_VARS += IMAGENAME USER_VARS += IMAGEPATH @@ -442,10 +442,10 @@ USER_VARS += NETMONPATH USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH -USER_VARS += QEMUPATHLIST +USER_VARS += QEMUVALIDPATHS USER_VARS += QEMUVIRTIOFSCMD USER_VARS += QEMUVIRTIOFSPATH -USER_VARS += QEMUVIRTIOFSPATHLIST +USER_VARS += QEMUVALIDVIRTIOFSPATHS USER_VARS += SHAREDIR USER_VARS += SHIMPATH USER_VARS += SYSCONFDIR @@ -468,7 +468,7 @@ USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS USER_VARS += DEFSHAREDFS USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON -USER_VARS += DEFVIRTIOFSDAEMONLIST +USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS USER_VARS += DEFVIRTIOFSCACHESIZE USER_VARS += DEFVIRTIOFSCACHE USER_VARS += DEFVIRTIOFSEXTRAARGS @@ -477,9 +477,9 @@ USER_VARS += DEFENABLEMEMPREALLOC USER_VARS += DEFENABLEHUGEPAGES USER_VARS += DEFENABLEVHOSTUSERSTORE USER_VARS += DEFVHOSTUSERSTOREPATH -USER_VARS += DEFVHOSTUSERSTOREPATHLIST +USER_VARS += DEFVALIDVHOSTUSERSTOREPATHS USER_VARS += DEFFILEMEMBACKEND -USER_VARS += DEFFILEMEMBACKENDLIST +USER_VARS += DEFVALIDFILEMEMBACKENDS USER_VARS += DEFENABLESWAP USER_VARS += DEFENABLEDEBUG USER_VARS += DEFDISABLENESTINGCHECKS @@ -629,15 +629,15 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ -e "s|@FCPATH@|$(FCPATH)|g" \ - -e "s|@FCPATHLIST@|$(FCPATHLIST)|g" \ + -e "s|@FCVALIDHYPERVISORPATHS@|$(FCVALIDHYPERVISORPATHS)|g" \ -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ - -e "s|@FCJAILERPATHLIST@|$(FCJAILERPATHLIST)|g" \ + -e "s|@FCVALIDJAILERPATHS@|$(FCVALIDJAILERPATHS)|g" \ -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ - -e "s|@ACRNPATHLIST@|$(ACRNPATHLIST)|g" \ + -e "s|@ACRNVALIDHYPERVISORPATHS@|$(ACRNVALIDHYPERVISORPATHS)|g" \ -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ - -e "s|@ACRNCTLPATHLIST@|$(ACRNCTLPATHLIST)|g" \ + -e "s|@ACRNVALIDCTLPATHS@|$(ACRNVALIDCTLPATHS)|g" \ -e "s|@CLHPATH@|$(CLHPATH)|g" \ - -e "s|@CLHPATHLIST@|$(CLHPATHLIST)|g" \ + -e "s|@CLHVALIDHYPERVISORPATHS@|$(CLHVALIDHYPERVISORPATHS)|g" \ -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ @@ -663,9 +663,9 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@PROJECT_TAG@|$(PROJECT_TAG)|g" \ -e "s|@PROJECT_TYPE@|$(PROJECT_TYPE)|g" \ -e "s|@QEMUPATH@|$(QEMUPATH)|g" \ - -e "s|@QEMUPATHLIST@|$(QEMUPATHLIST)|g" \ + -e "s|@QEMUVALIDHYPERVISORPATHS@|$(QEMUVALIDHYPERVISORPATHS)|g" \ -e "s|@QEMUVIRTIOFSPATH@|$(QEMUVIRTIOFSPATH)|g" \ - -e "s|@QEMUVIRTIOFSPATHLIST@|$(QEMUVIRTIOFSPATHLIST)|g" \ + -e "s|@QEMUVALIDVIRTIOFSPATHS@|$(QEMUVALIDVIRTIOFSPATHS)|g" \ -e "s|@RUNTIME_NAME@|$(TARGET)|g" \ -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ @@ -690,7 +690,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \ -e "s|@DEFSHAREDFS_QEMU_VIRTIOFS@|$(DEFSHAREDFS_QEMU_VIRTIOFS)|g" \ -e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \ - -e "s|@DEFVIRTIOFSDAEMONLIST@|$(DEFVIRTIOFSDAEMONLIST)|g" \ + -e "s|@DEFVALIDVIRTIOFSDAEMONPATHS@|$(DEFVALIDVIRTIOFSDAEMONPATHS)|g" \ -e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \ -e "s|@DEFVIRTIOFSCACHE@|$(DEFVIRTIOFSCACHE)|g" \ -e "s|@DEFVIRTIOFSEXTRAARGS@|$(DEFVIRTIOFSEXTRAARGS)|g" \ @@ -699,9 +699,9 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit -e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \ -e "s|@DEFENABLEVHOSTUSERSTORE@|$(DEFENABLEVHOSTUSERSTORE)|g" \ -e "s|@DEFVHOSTUSERSTOREPATH@|$(DEFVHOSTUSERSTOREPATH)|g" \ - -e "s|@DEFVHOSTUSERSTOREPATHLIST@|$(DEFVHOSTUSERSTOREPATHLIST)|g" \ + -e "s|@DEFVALIDVHOSTUSERSTOREPATHS@|$(DEFVALIDVHOSTUSERSTOREPATHS)|g" \ -e "s|@DEFFILEMEMBACKEND@|$(DEFFILEMEMBACKEND)|g" \ - -e "s|@DEFFILEMEMBACKENDLIST@|$(DEFFILEMEMBACKENDLIST)|g" \ + -e "s|@DEFVALIDFILEMEMBACKENDS@|$(DEFVALIDFILEMEMBACKENDS)|g" \ -e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \ -e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \ -e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \ diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 85d60cd443..6ffa8d5d7a 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -24,13 +24,13 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @ACRNPATHLIST@ -path_list = @ACRNPATHLIST@ +# Your distribution recommends: @ACRNVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @ACRNVALIDHYPERVISORPATHS@ # List of valid annotations values for ctlpath # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @ACRNCTLPATHLIST@ -ctlpath_list = @ACRNCTLPATHLIST@ +# Your distribution recommends: @ACRNVALIDCTLPATHS@ +valid_ctlpaths = @ACRNVALIDCTLPATHS@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 33379c8852..4cc528d564 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -23,8 +23,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @CLHPATHLIST@ -path_list = @CLHPATHLIST@ +# Your distribution recommends: @CLHVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -75,8 +75,8 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ -virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ +# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@ +valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index e21d2c1ddc..e6a2b823b7 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -23,8 +23,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @FCPATHLIST@ -path_list = @FCPATHLIST@ +# Your distribution recommends: @FCVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @FCVALIDHYPERVISORPATHS@ # Path for the jailer specific to firecracker # If the jailer path is not set kata will launch firecracker @@ -37,8 +37,8 @@ path_list = @FCPATHLIST@ # List of valid jailer path values for the hypervisor # Each member of the list can be a regular expression # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @FCJAILERPATHLIST@ -jailer_path_list = @FCJAILERPATHLIST@ +# Your distribution recommends: @FCVALIDJAILERPATHS@ +valid_jailer_paths = @FCVALIDJAILERPATHS@ # Optional space-separated list of options to pass to the guest kernel. diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index 27c963d749..41186907dd 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -26,8 +26,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @QEMUVIRTIOFSPATHLIST@ -path_list = @QEMUVIRTIOFSPATHLIST@ +# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -125,8 +125,8 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ -virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ +# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@ +valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -219,8 +219,8 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVHOSTUSERSTOREPATHLIST@ -vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ +# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ +valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@ # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled @@ -230,8 +230,8 @@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # List of valid annotations values for the file_mem_backend annotation # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFFILEMEMBACKENDLIST@ -file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ +# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ +valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 87b10d3175..0e6051430a 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -24,8 +24,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @QEMUPATHLIST@ -path_list = @QEMUPATHLIST@ +# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having @@ -129,8 +129,8 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" # List of valid annotations values for the virtiofs daemon # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVIRTIOFSDAEMONLIST@ -virtio_fs_daemon_list = @DEFVIRTIOFSDAEMONLIST@ +# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@ +valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ @@ -223,8 +223,8 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # List of valid annotations values for the vhost user store path # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVHOSTUSERSTOREPATHLIST@ -vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ +# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ +valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@ # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled @@ -234,8 +234,8 @@ vhost_user_store_path_list = @DEFVHOSTUSERSTOREPATHLIST@ # List of valid annotations values for the file_mem_backend annotation # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFFILEMEMBACKENDLIST@ -file_mem_backend_list = @DEFFILEMEMBACKENDLIST@ +# Your distribution recommends: @DEFVALIDFILEMEMBACKENDS@ +valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index ba9f841dee..ca857ca97b 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -71,12 +71,12 @@ type factory struct { type hypervisor struct { Path string `toml:"path"` - HypervisorPathList []string `toml:"path_list"` + HypervisorPathList []string `toml:"valid_hypervisor_paths"` JailerPath string `toml:"jailer_path"` - JailerPathList []string `toml:"jailer_path_list"` + JailerPathList []string `toml:"valid_jailer_paths"` Kernel string `toml:"kernel"` CtlPath string `toml:"ctlpath"` - CtlPathList []string `toml:"ctlpath_list"` + CtlPathList []string `toml:"valid_ctlpaths"` Initrd string `toml:"initrd"` Image string `toml:"image"` Firmware string `toml:"firmware"` @@ -88,7 +88,7 @@ type hypervisor struct { EntropySource string `toml:"entropy_source"` SharedFS string `toml:"shared_fs"` VirtioFSDaemon string `toml:"virtio_fs_daemon"` - VirtioFSDaemonList []string `toml:"virtio_fs_daemon_list"` + VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` VirtioFSCache string `toml:"virtio_fs_cache"` VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` @@ -97,7 +97,7 @@ type hypervisor struct { BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` EnableVhostUserStore bool `toml:"enable_vhost_user_store"` VhostUserStorePath string `toml:"vhost_user_store_path"` - VhostUserStorePathList []string `toml:"vhost_user_store_path_list"` + VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` NumVCPUs int32 `toml:"default_vcpus"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` @@ -113,7 +113,7 @@ type hypervisor struct { IOMMU bool `toml:"enable_iommu"` IOMMUPlatform bool `toml:"enable_iommu_platform"` FileBackedMemRootDir string `toml:"file_mem_backend"` - FileBackedMemRootList []string `toml:"file_mem_backend_list"` + FileBackedMemRootList []string `toml:"valid_file_mem_backends"` Swap bool `toml:"enable_swap"` Debug bool `toml:"enable_debug"` DisableNestingChecks bool `toml:"disable_nesting_checks"` From 451608fb281d27ae31738faf5a52d0699523d800 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Thu, 10 Sep 2020 21:50:24 +0200 Subject: [PATCH 088/124] makefile: Add missing generated vars to `USER_VARS` This was discovered while checking a massive change in variables. The root cause for the error is a very long list of manual replacements, that is best replaced with a $(foreach). All individual variables in the output configuration files were checked against the old build using diff. This is a forward port of a makefile fix included in PR https://github.com/kata-containers/runtime/issues/3004 for issue https://github.com/kata-containers/runtime/issues/2943 Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/Makefile | 122 +++++++++++-------------------------------- 1 file changed, 30 insertions(+), 92 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 8d657e7643..a5b8d4ef6f 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -95,6 +95,14 @@ COLLECT_SCRIPT = data/kata-collect-data.sh COLLECT_SCRIPT_SRC = $(COLLECT_SCRIPT).in GENERATED_FILES += $(COLLECT_SCRIPT) +GENERATED_VARS = \ + VERSION \ + CONFIG_ACRN_IN \ + CONFIG_QEMU_IN \ + CONFIG_QEMU_VIRTIOFS_IN \ + CONFIG_CLH_IN \ + CONFIG_FC_IN \ + $(USER_VARS) SCRIPTS += $(COLLECT_SCRIPT) SCRIPTS_DIR := $(BINDIR) @@ -396,17 +404,24 @@ SHAREDIR := $(SHAREDIR) # list of variables the user may wish to override USER_VARS += ARCH USER_VARS += BINDIR +USER_VARS += CONFIG_ACRN_IN +USER_VARS += CONFIG_CLH_IN +USER_VARS += CONFIG_FC_IN USER_VARS += CONFIG_PATH +USER_VARS += CONFIG_QEMU_IN +USER_VARS += CONFIG_QEMU_VIRTIOFS_IN USER_VARS += DESTDIR USER_VARS += DEFAULT_HYPERVISOR +USER_VARS += DEFENABLEMSWAP USER_VARS += ACRNCMD USER_VARS += ACRNCTLCMD USER_VARS += ACRNPATH -USER_VARS += ACRNPATHLIST +USER_VARS += ACRNVALIDHYPERVISORPATHS USER_VARS += ACRNCTLPATH USER_VARS += ACRNVALIDCTLPATHS USER_VARS += CLHPATH USER_VARS += CLHVALIDHYPERVISORPATHS +USER_VARS += FIRMWAREPATH_CLH USER_VARS += FCCMD USER_VARS += FCPATH USER_VARS += FCVALIDHYPERVISORPATHS @@ -423,6 +438,11 @@ USER_VARS += KERNELTYPE USER_VARS += KERNELTYPE_FC USER_VARS += KERNELTYPE_ACRN USER_VARS += KERNELTYPE_CLH +USER_VARS += KERNELPATH_ACRN +USER_VARS += KERNELPATH +USER_VARS += KERNELPATH_CLH +USER_VARS += KERNELPATH_FC +USER_VARS += KERNELVIRTIOFSPATH USER_VARS += FIRMWAREPATH USER_VARS += MACHINEACCELERATORS USER_VARS += CPUFEATURES @@ -435,17 +455,22 @@ USER_VARS += PKGLIBDIR USER_VARS += PKGLIBEXECDIR USER_VARS += PKGRUNDIR USER_VARS += PREFIX +USER_VARS += PROJECT_BUG_URL USER_VARS += PROJECT_NAME +USER_VARS += PROJECT_ORG USER_VARS += PROJECT_PREFIX +USER_VARS += PROJECT_TAG USER_VARS += PROJECT_TYPE +USER_VARS += PROJECT_URL USER_VARS += NETMONPATH USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH -USER_VARS += QEMUVALIDPATHS +USER_VARS += QEMUVALIDHYPERVISORPATHS USER_VARS += QEMUVIRTIOFSCMD USER_VARS += QEMUVIRTIOFSPATH USER_VARS += QEMUVALIDVIRTIOFSPATHS +USER_VARS += RUNTIME_NAME USER_VARS += SHAREDIR USER_VARS += SHIMPATH USER_VARS += SYSCONFDIR @@ -456,6 +481,7 @@ USER_VARS += DEFMEMSZ USER_VARS += DEFMEMSLOTS USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL_ACRN +USER_VARS += DEFNETWORKMODEL_CLH USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU USER_VARS += DEFDISABLEGUESTSECCOMP @@ -472,6 +498,7 @@ USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS USER_VARS += DEFVIRTIOFSCACHESIZE USER_VARS += DEFVIRTIOFSCACHE USER_VARS += DEFVIRTIOFSEXTRAARGS +USER_VARS += DEFENABLEANNOTATIONS USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFENABLEMEMPREALLOC USER_VARS += DEFENABLEHUGEPAGES @@ -621,96 +648,7 @@ GENERATED_FILES += $(CONFIGS) $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit $(QUIET_GENERATE)$(SED) \ -e "s|@COMMIT@|$(shell cat .git-commit)|g" \ - -e "s|@VERSION@|$(VERSION)|g" \ - -e "s|@CONFIG_ACRN_IN@|$(CONFIG_ACRN_IN)|g" \ - -e "s|@CONFIG_QEMU_IN@|$(CONFIG_QEMU_IN)|g" \ - -e "s|@CONFIG_QEMU_VIRTIOFS_IN@|$(CONFIG_QEMU_VIRTIOFS_IN)|g" \ - -e "s|@CONFIG_CLH_IN@|$(CONFIG_CLH_IN)|g" \ - -e "s|@CONFIG_FC_IN@|$(CONFIG_FC_IN)|g" \ - -e "s|@CONFIG_PATH@|$(CONFIG_PATH)|g" \ - -e "s|@FCPATH@|$(FCPATH)|g" \ - -e "s|@FCVALIDHYPERVISORPATHS@|$(FCVALIDHYPERVISORPATHS)|g" \ - -e "s|@FCJAILERPATH@|$(FCJAILERPATH)|g" \ - -e "s|@FCVALIDJAILERPATHS@|$(FCVALIDJAILERPATHS)|g" \ - -e "s|@ACRNPATH@|$(ACRNPATH)|g" \ - -e "s|@ACRNVALIDHYPERVISORPATHS@|$(ACRNVALIDHYPERVISORPATHS)|g" \ - -e "s|@ACRNCTLPATH@|$(ACRNCTLPATH)|g" \ - -e "s|@ACRNVALIDCTLPATHS@|$(ACRNVALIDCTLPATHS)|g" \ - -e "s|@CLHPATH@|$(CLHPATH)|g" \ - -e "s|@CLHVALIDHYPERVISORPATHS@|$(CLHVALIDHYPERVISORPATHS)|g" \ - -e "s|@SYSCONFIG@|$(SYSCONFIG)|g" \ - -e "s|@IMAGEPATH@|$(IMAGEPATH)|g" \ - -e "s|@KERNELPATH_ACRN@|$(KERNELPATH_ACRN)|g" \ - -e "s|@KERNELPATH_FC@|$(KERNELPATH_FC)|g" \ - -e "s|@KERNELPATH_CLH@|$(KERNELPATH_CLH)|g" \ - -e "s|@KERNELPATH@|$(KERNELPATH)|g" \ - -e "s|@KERNELVIRTIOFSPATH@|$(KERNELVIRTIOFSPATH)|g" \ - -e "s|@INITRDPATH@|$(INITRDPATH)|g" \ - -e "s|@FIRMWAREPATH@|$(FIRMWAREPATH)|g" \ - -e "s|@MACHINEACCELERATORS@|$(MACHINEACCELERATORS)|g" \ - -e "s|@CPUFEATURES@|$(CPUFEATURES)|g" \ - -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ - -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ - -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ - -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ - -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ - -e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \ - -e "s|@NETMONPATH@|$(NETMONPATH)|g" \ - -e "s|@PROJECT_BUG_URL@|$(PROJECT_BUG_URL)|g" \ - -e "s|@PROJECT_ORG@|$(PROJECT_ORG)|g" \ - -e "s|@PROJECT_URL@|$(PROJECT_URL)|g" \ - -e "s|@PROJECT_NAME@|$(PROJECT_NAME)|g" \ - -e "s|@PROJECT_TAG@|$(PROJECT_TAG)|g" \ - -e "s|@PROJECT_TYPE@|$(PROJECT_TYPE)|g" \ - -e "s|@QEMUPATH@|$(QEMUPATH)|g" \ - -e "s|@QEMUVALIDHYPERVISORPATHS@|$(QEMUVALIDHYPERVISORPATHS)|g" \ - -e "s|@QEMUVIRTIOFSPATH@|$(QEMUVIRTIOFSPATH)|g" \ - -e "s|@QEMUVALIDVIRTIOFSPATHS@|$(QEMUVALIDVIRTIOFSPATHS)|g" \ - -e "s|@RUNTIME_NAME@|$(TARGET)|g" \ - -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ - -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ - -e "s|@DEFVCPUS@|$(DEFVCPUS)|g" \ - -e "s|@DEFMAXVCPUS@|$(DEFMAXVCPUS)|g" \ - -e "s|@DEFMAXVCPUS_ACRN@|$(DEFMAXVCPUS_ACRN)|g" \ - -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ - -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ - -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ - -e "s|@DEFNETWORKMODEL_ACRN@|$(DEFNETWORKMODEL_ACRN)|g" \ - -e "s|@DEFNETWORKMODEL_CLH@|$(DEFNETWORKMODEL_CLH)|g" \ - -e "s|@DEFNETWORKMODEL_FC@|$(DEFNETWORKMODEL_FC)|g" \ - -e "s|@DEFNETWORKMODEL_QEMU@|$(DEFNETWORKMODEL_QEMU)|g" \ - -e "s|@DEFENABLEANNOTATIONS@|$(DEFENABLEANNOTATIONS)|g" \ - -e "s|@DEFDISABLEGUESTSECCOMP@|$(DEFDISABLEGUESTSECCOMP)|g" \ - -e "s|@DEFAULTEXPFEATURES@|$(DEFAULTEXPFEATURES)|g" \ - -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ - -e "s|@DEFBLOCKSTORAGEDRIVER_ACRN@|$(DEFBLOCKSTORAGEDRIVER_ACRN)|g" \ - -e "s|@DEFBLOCKSTORAGEDRIVER_FC@|$(DEFBLOCKSTORAGEDRIVER_FC)|g" \ - -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU@|$(DEFBLOCKSTORAGEDRIVER_QEMU)|g" \ - -e "s|@DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS@|$(DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS)|g" \ - -e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \ - -e "s|@DEFSHAREDFS_QEMU_VIRTIOFS@|$(DEFSHAREDFS_QEMU_VIRTIOFS)|g" \ - -e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \ - -e "s|@DEFVALIDVIRTIOFSDAEMONPATHS@|$(DEFVALIDVIRTIOFSDAEMONPATHS)|g" \ - -e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \ - -e "s|@DEFVIRTIOFSCACHE@|$(DEFVIRTIOFSCACHE)|g" \ - -e "s|@DEFVIRTIOFSEXTRAARGS@|$(DEFVIRTIOFSEXTRAARGS)|g" \ - -e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \ - -e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \ - -e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \ - -e "s|@DEFENABLEVHOSTUSERSTORE@|$(DEFENABLEVHOSTUSERSTORE)|g" \ - -e "s|@DEFVHOSTUSERSTOREPATH@|$(DEFVHOSTUSERSTOREPATH)|g" \ - -e "s|@DEFVALIDVHOSTUSERSTOREPATHS@|$(DEFVALIDVHOSTUSERSTOREPATHS)|g" \ - -e "s|@DEFFILEMEMBACKEND@|$(DEFFILEMEMBACKEND)|g" \ - -e "s|@DEFVALIDFILEMEMBACKENDS@|$(DEFVALIDFILEMEMBACKENDS)|g" \ - -e "s|@DEFENABLEMSWAP@|$(DEFENABLESWAP)|g" \ - -e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \ - -e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \ - -e "s|@DEFMSIZE9P@|$(DEFMSIZE9P)|g" \ - -e "s|@DEFHOTPLUGVFIOONROOTBUS@|$(DEFHOTPLUGVFIOONROOTBUS)|g" \ - -e "s|@DEFPCIEROOTPORT@|$(DEFPCIEROOTPORT)|g" \ - -e "s|@DEFENTROPYSOURCE@|$(DEFENTROPYSOURCE)|g" \ - -e "s|@DEFSANDBOXCGROUPONLY@|$(DEFSANDBOXCGROUPONLY)|g" \ - -e "s|@FEATURE_SELINUX@|$(FEATURE_SELINUX)|g" \ + $(foreach v,$(GENERATED_VARS),-e "s|@$v@|$($v)|g") \ $< > $@ generate-config: $(CONFIGS) From c6afad2a06cf3154c170d693defc90f2c4601baa Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 11 Sep 2020 11:06:26 +0200 Subject: [PATCH 089/124] annotations: Add unit test for regexpContains function James O.D Hunt: "But also, regexpContains() and checkPathIsInGlobList() seem like good candidates for some unit tests. The "look" obvious, but a few boundary condition tests would be useful I think (filenames with spaces, backslashes, special characters, and relative & absolute paths are also an interesting thought here)." There aren't that many boundary conditions on a list with regexps, if you assume the regexp match function itself works. However, the tests is useful in documenting expectations. Fixes: #901 Suggested-by: James O.D. Hunt Signed-off-by: Christophe de Dinechin --- .../virtcontainers/pkg/oci/utils_test.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index 533ee2ec71..a9109862bc 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -893,6 +893,34 @@ func TestAddRuntimeAnnotations(t *testing.T) { assert.Equal(config.NetworkConfig.InterworkingModel, vc.NetXConnectMacVtapModel) } +func TestRegexpContains(t *testing.T) { + assert := assert.New(t) + + type testData struct { + regexps []string + toMatch string + expected bool + } + + data := []testData{ + {[]string{}, "", false}, + {[]string{}, "nonempty", false}, + {[]string{"simple"}, "simple", true}, + {[]string{"simple"}, "some_simple_text", true}, + {[]string{"simple"}, "simp", false}, + {[]string{"one", "two"}, "one", true}, + {[]string{"one", "two"}, "two", true}, + {[]string{"o*"}, "oooo", true}, + {[]string{"o*"}, "oooa", true}, + {[]string{"^o*$"}, "oooa", false}, + } + + for _, d := range data { + matched := regexpContains(d.regexps, d.toMatch) + assert.Equal(d.expected, matched, "%+v", d) + } +} + func TestIsCRIOContainerManager(t *testing.T) { assert := assert.New(t) From b6059f35660fcdd6426e53fa0cbcdf3a223cf048 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 11 Sep 2020 12:59:53 +0200 Subject: [PATCH 090/124] annotations: Add unit test for checkPathIsInGlobs There are a few interesting corner cases to consider for this function. Fixes: #901 Suggested-by: James O.D. Hunt Signed-off-by: Christophe de Dinechin --- .../virtcontainers/pkg/oci/utils_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index a9109862bc..5ea0dedd7d 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -921,6 +921,37 @@ func TestRegexpContains(t *testing.T) { } } +func TestCheckPathIsInGlobs(t *testing.T) { + assert := assert.New(t) + + type testData struct { + globs []string + toMatch string + expected bool + } + + data := []testData{ + {[]string{}, "", false}, + {[]string{}, "nonempty", false}, + {[]string{"simple"}, "simple", false}, + {[]string{"simple"}, "some_simple_text", false}, + {[]string{"/bin/ls"}, "/bin/ls", true}, + {[]string{"/bin/ls", "/bin/false"}, "/bin/ls", true}, + {[]string{"/bin/ls", "/bin/false"}, "/bin/false", true}, + {[]string{"/bin/ls", "/bin/false"}, "/bin/bar", false}, + {[]string{"/bin/*ls*"}, "/bin/ls", true}, + {[]string{"/bin/*ls*"}, "/bin/false", true}, + {[]string{"bin/ls"}, "/bin/ls", false}, + {[]string{"./bin/ls"}, "/bin/ls", false}, + {[]string{"*/bin/ls"}, "/bin/ls", false}, + } + + for _, d := range data { + matched := checkPathIsInGlobs(d.globs, d.toMatch) + assert.Equal(d.expected, matched, "%+v", d) + } +} + func TestIsCRIOContainerManager(t *testing.T) { assert := assert.New(t) From 0cc6297716371428493218428e3c27990c68f5a4 Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 11 Sep 2020 12:52:10 +0200 Subject: [PATCH 091/124] annotations: Split addHypervisorOverrides to reduce complexity Warning from gocyclo during make check: virtcontainers/pkg/oci/utils.go:404:1: cyclomatic complexity 37 of func `addHypervisorConfigOverrides` is high (> 30) (gocyclo) func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, runtime RuntimeConfig) error { ^ Fixes: #901 Signed-off-by: Christophe de Dinechin --- src/runtime/virtcontainers/pkg/oci/utils.go | 67 ++++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 1dd9942c0c..c417c7121a 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -415,36 +415,8 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return err } - if value, ok := ocispec.Annotations[vcAnnotations.HypervisorPath]; ok { - if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorPathList, value) { - return fmt.Errorf("hypervisor %v required from annotation is not valid", value) - } - config.HypervisorConfig.HypervisorPath = value - } - - if value, ok := ocispec.Annotations[vcAnnotations.JailerPath]; ok { - if !checkPathIsInGlobs(runtime.HypervisorConfig.JailerPathList, value) { - return fmt.Errorf("jailer %v required from annotation is not valid", value) - } - config.HypervisorConfig.JailerPath = value - } - - if value, ok := ocispec.Annotations[vcAnnotations.CtlPath]; ok { - if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorCtlPathList, value) { - return fmt.Errorf("hypervisor control %v required from annotation is not valid", value) - } - config.HypervisorConfig.HypervisorCtlPath = value - } - - if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { - if value != "" { - params := vc.DeserializeParams(strings.Fields(value)) - for _, param := range params { - if err := config.HypervisorConfig.AddKernelParam(param); err != nil { - return fmt.Errorf("Error adding kernel parameters in annotation kernel_params : %v", err) - } - } - } + if err := addHypervisorPathOverrides(ocispec, config, runtime); err != nil { + return err } if value, ok := ocispec.Annotations[vcAnnotations.MachineType]; ok { @@ -507,6 +479,41 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig, return nil } +func addHypervisorPathOverrides(ocispec specs.Spec, config *vc.SandboxConfig, runtime RuntimeConfig) error { + if value, ok := ocispec.Annotations[vcAnnotations.HypervisorPath]; ok { + if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorPathList, value) { + return fmt.Errorf("hypervisor %v required from annotation is not valid", value) + } + config.HypervisorConfig.HypervisorPath = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.JailerPath]; ok { + if !checkPathIsInGlobs(runtime.HypervisorConfig.JailerPathList, value) { + return fmt.Errorf("jailer %v required from annotation is not valid", value) + } + config.HypervisorConfig.JailerPath = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.CtlPath]; ok { + if !checkPathIsInGlobs(runtime.HypervisorConfig.HypervisorCtlPathList, value) { + return fmt.Errorf("hypervisor control %v required from annotation is not valid", value) + } + config.HypervisorConfig.HypervisorCtlPath = value + } + + if value, ok := ocispec.Annotations[vcAnnotations.KernelParams]; ok { + if value != "" { + params := vc.DeserializeParams(strings.Fields(value)) + for _, param := range params { + if err := config.HypervisorConfig.AddKernelParam(param); err != nil { + return fmt.Errorf("Error adding kernel parameters in annotation kernel_params : %v", err) + } + } + } + } + return nil +} + func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, runtime RuntimeConfig) error { if value, ok := ocispec.Annotations[vcAnnotations.DefaultMemory]; ok { memorySz, err := strconv.ParseUint(value, 10, 32) From 8a364d21450e2f9997eafe79ac556558ecc65aac Mon Sep 17 00:00:00 2001 From: Christophe de Dinechin Date: Fri, 11 Sep 2020 14:13:45 +0200 Subject: [PATCH 092/124] annotations: Correct unit tests to validate new protections Add the verification of some basic protections, namely that: - EnableAnnotations is honored - Dangerous paths cannot be modified if no match - Errors are returned when expected Fixes: #901 Signed-off-by: Christophe de Dinechin --- .../virtcontainers/pkg/oci/utils_test.go | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index 5ea0dedd7d..af4b82545b 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -681,7 +681,20 @@ func TestAddAssetAnnotations(t *testing.T) { Console: consolePath, } - addAnnotations(ocispec, &config, runtimeConfig) + // Try annotations without enabling them first + err := addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + assert.Exactly(map[string]string{}, config.Annotations) + + // Check if annotation not enabled correctly + runtimeConfig.HypervisorConfig.EnableAnnotations = []string{"nonexistent"} + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + + // Check that it works if all annotation are enabled + runtimeConfig.HypervisorConfig.EnableAnnotations = []string{".*"} + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.NoError(err) assert.Exactly(expectedAnnotations, config.Annotations) } @@ -771,6 +784,9 @@ func TestAddHypervisorAnnotations(t *testing.T) { HypervisorType: vc.QemuHypervisor, Console: consolePath, } + runtimeConfig.HypervisorConfig.EnableAnnotations = []string{".*"} + runtimeConfig.HypervisorConfig.FileBackedMemRootList = []string{"/dev/shm*"} + runtimeConfig.HypervisorConfig.VirtioFSDaemonList = []string{"/bin/*ls*"} ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" addHypervisorConfigOverrides(ocispec, &config, runtimeConfig) @@ -794,7 +810,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.BlockDeviceCacheDirect] = "true" ocispec.Annotations[vcAnnotations.BlockDeviceCacheNoflush] = "true" ocispec.Annotations[vcAnnotations.SharedFS] = "virtio-fs" - ocispec.Annotations[vcAnnotations.VirtioFSDaemon] = "/home/virtiofsd" + ocispec.Annotations[vcAnnotations.VirtioFSDaemon] = "/bin/false" ocispec.Annotations[vcAnnotations.VirtioFSCache] = "/home/cache" ocispec.Annotations[vcAnnotations.Msize9p] = "512" ocispec.Annotations[vcAnnotations.MachineType] = "q35" @@ -830,7 +846,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.BlockDeviceCacheDirect, true) assert.Equal(config.HypervisorConfig.BlockDeviceCacheNoflush, true) assert.Equal(config.HypervisorConfig.SharedFS, "virtio-fs") - assert.Equal(config.HypervisorConfig.VirtioFSDaemon, "/home/virtiofsd") + assert.Equal(config.HypervisorConfig.VirtioFSDaemon, "/bin/false") assert.Equal(config.HypervisorConfig.VirtioFSCache, "/home/cache") assert.Equal(config.HypervisorConfig.Msize9p, uint32(512)) assert.Equal(config.HypervisorConfig.HypervisorMachineType, "q35") @@ -865,6 +881,67 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Error(err) } +func TestAddProtectedHypervisorAnnotations(t *testing.T) { + assert := assert.New(t) + + config := vc.SandboxConfig{ + Annotations: make(map[string]string), + } + + ocispec := specs.Spec{ + Annotations: make(map[string]string), + } + + runtimeConfig := RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + Console: consolePath, + } + ocispec.Annotations[vcAnnotations.KernelParams] = "vsyscall=emulate iommu=on" + err := addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + assert.Exactly(vc.HypervisorConfig{}, config.HypervisorConfig) + + // Enable annotations + runtimeConfig.HypervisorConfig.EnableAnnotations = []string{".*"} + + ocispec.Annotations[vcAnnotations.FileBackedMemRootDir] = "/dev/shm" + ocispec.Annotations[vcAnnotations.VirtioFSDaemon] = "/bin/false" + + config.HypervisorConfig.FileBackedMemRootDir = "do-not-touch" + config.HypervisorConfig.VirtioFSDaemon = "dangerous-daemon" + + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + assert.Equal(config.HypervisorConfig.FileBackedMemRootDir, "do-not-touch") + assert.Equal(config.HypervisorConfig.VirtioFSDaemon, "dangerous-daemon") + + // Now enable them and check again + runtimeConfig.HypervisorConfig.FileBackedMemRootList = []string{"/dev/*m"} + runtimeConfig.HypervisorConfig.VirtioFSDaemonList = []string{"/bin/*ls*"} + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.NoError(err) + assert.Equal(config.HypervisorConfig.FileBackedMemRootDir, "/dev/shm") + assert.Equal(config.HypervisorConfig.VirtioFSDaemon, "/bin/false") + + // In case an absurd large value is provided, the config value if not over-ridden + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "655536" + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "-1" + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + + ocispec.Annotations[vcAnnotations.DefaultVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "-1" + err = addAnnotations(ocispec, &config, runtimeConfig) + assert.Error(err) + + ocispec.Annotations[vcAnnotations.DefaultMaxVCPUs] = "1" + ocispec.Annotations[vcAnnotations.DefaultMemory] = fmt.Sprintf("%d", vc.MinHypervisorMemory+1) + assert.Error(err) +} + func TestAddRuntimeAnnotations(t *testing.T) { assert := assert.New(t) From cc8ec7b0e96104107f5e6b59d6a5e356483e23e5 Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Wed, 14 Oct 2020 15:49:03 -0400 Subject: [PATCH 093/124] versions: Update Kubernetes, containerd, cri-o and cri-tools Kubernetes: from 1.17.3 to 1.18.9 CRI-O: from 0eec454168e381e460b3d6de07bf50bfd9b0d082 (1.17) to 1.18.3 Containerd: from 3a4acfbc99aa976849f51a8edd4af20ead51d8d7 (1.3.3) to 1.3.7 cri-tools: from 1.17.0 to 1.18.0 Fixes: #960. Signed-off-by: Salvador Fuentes --- versions.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versions.yaml b/versions.yaml index 2ac20ba802..eb5982ca87 100644 --- a/versions.yaml +++ b/versions.yaml @@ -179,7 +179,7 @@ externals: description: | OCI-based Kubernetes Container Runtime Interface implementation url: "https://github.com/cri-o/cri-o" - version: "0eec454168e381e460b3d6de07bf50bfd9b0d082" + version: "v1.18.3" meta: openshift: "6273bea4c9ed788aeb3d051ebf2d030060c05b6c" crictl: 1.0.0-beta.2 @@ -191,12 +191,12 @@ externals: tarball_url: "https://storage.googleapis.com/cri-containerd-release" # Next commit from 1.3 branch contains fix to be able to run # tests using go 1.13 - version: "3a4acfbc99aa976849f51a8edd4af20ead51d8d7" + version: "v1.3.7" critools: description: "CLI tool for Container Runtime Interface (CRI)" url: "https://github.com/kubernetes-sigs/cri-tools" - version: "1.17.0" + version: "1.18.0" docker: description: "Moby project container manager" @@ -214,7 +214,7 @@ externals: uscan-url: >- https://github.com/kubernetes/kubernetes/tags .*/v?([\d\.]+)\.tar\.gz - version: "1.17.3-00" + version: "1.18.9-00" openshift: description: | From ba069f9baac1989a6034aae6c254885be3343efd Mon Sep 17 00:00:00 2001 From: bin liu Date: Wed, 14 Oct 2020 20:51:05 +0800 Subject: [PATCH 094/124] rustjail: add length check for uid_mappings in rootless euid mapping This might be a copy miss, gid_mappings is checked twice, one should be uid_mappings. Fixes: #952 Signed-off-by: bin liu --- src/agent/rustjail/src/validator.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent/rustjail/src/validator.rs b/src/agent/rustjail/src/validator.rs index deaf7c14ac..4e3ce43182 100644 --- a/src/agent/rustjail/src/validator.rs +++ b/src/agent/rustjail/src/validator.rs @@ -225,7 +225,8 @@ fn rootless_euid_mapping(oci: &Spec) -> Result<()> { return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); } - if linux.gid_mappings.len() == 0 || linux.gid_mappings.len() == 0 { + if linux.uid_mappings.len() == 0 || linux.gid_mappings.len() == 0 { + // rootless containers requires at least one UID/GID mapping return Err(anyhow!(nix::Error::from_errno(Errno::EINVAL))); } From e733c13cf7fb07848bfa4b9600f243d36c1161d8 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 14:27:29 +0800 Subject: [PATCH 095/124] agent: replace `if let Err` with `map_err` Fixes #934 Signed-off-by: Tim Zhang --- src/agent/rustjail/src/capabilities.rs | 7 ++- src/agent/rustjail/src/container.rs | 43 +++++++--------- src/agent/src/main.rs | 12 ++--- src/agent/src/mount.rs | 5 +- src/agent/src/namespace.rs | 10 ++-- src/agent/src/rpc.rs | 70 ++++++++++---------------- src/agent/src/uevent.rs | 6 +-- 7 files changed, 62 insertions(+), 91 deletions(-) diff --git a/src/agent/rustjail/src/capabilities.rs b/src/agent/rustjail/src/capabilities.rs index f9203efe1b..91f6ea823c 100644 --- a/src/agent/rustjail/src/capabilities.rs +++ b/src/agent/rustjail/src/capabilities.rs @@ -126,13 +126,12 @@ pub fn drop_privileges(cfd_log: RawFd, caps: &LinuxCapabilities) -> Result<()> { ) .map_err(|e| anyhow!(e.to_string()))?; - if let Err(_) = caps::set( + let _ = caps::set( None, CapSet::Ambient, to_capshashset(cfd_log, caps.ambient.as_ref()), - ) { - log_child!(cfd_log, "failed to set ambient capability"); - } + ) + .map_err(|_| log_child!(cfd_log, "failed to set ambient capability")); Ok(()) } diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 377c8124a0..91657e322a 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use dirs; use lazy_static; use libc::pid_t; @@ -457,9 +457,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { // Ref: https://github.com/opencontainers/runc/commit/50a19c6ff828c58e5dab13830bd3dacde268afe5 // if !nses.is_empty() { - if let Err(e) = prctl::set_dumpable(false) { - return Err(anyhow!(e).context("set process non-dumpable failed")); - }; + prctl::set_dumpable(false) + .map_err(|e| anyhow!(e).context("set process non-dumpable failed"))?; } if userns { @@ -590,9 +589,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { // NoNewPeiviledges, Drop capabilities if oci_process.no_new_privileges { - if let Err(_) = prctl::set_no_new_privileges(true) { - return Err(anyhow!("cannot set no new privileges")); - } + prctl::set_no_new_privileges(true).map_err(|_| anyhow!("cannot set no new privileges"))?; } if oci_process.capabilities.is_some() { @@ -1074,14 +1071,12 @@ fn do_exec(args: &[String]) -> ! { .collect(); let a: Vec<&CStr> = sa.iter().map(|s| s.as_c_str()).collect(); - if let Err(e) = unistd::execvp(p.as_c_str(), a.as_slice()) { - match e { - nix::Error::Sys(errno) => { - std::process::exit(errno as i32); - } - _ => std::process::exit(-2), + let _ = unistd::execvp(p.as_c_str(), a.as_slice()).map_err(|e| match e { + nix::Error::Sys(errno) => { + std::process::exit(errno as i32); } - } + _ => std::process::exit(-2), + }); unreachable!() } @@ -1291,9 +1286,9 @@ fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIDMapping]) -> Resul fn setid(uid: Uid, gid: Gid) -> Result<()> { // set uid/gid - if let Err(e) = prctl::set_keep_capabilities(true) { - bail!(anyhow!(e).context("set keep capabilities returned")); - }; + prctl::set_keep_capabilities(true) + .map_err(|e| anyhow!(e).context("set keep capabilities returned"))?; + { unistd::setresgid(gid, gid, gid)?; } @@ -1305,9 +1300,9 @@ fn setid(uid: Uid, gid: Gid) -> Result<()> { capabilities::reset_effective()?; } - if let Err(e) = prctl::set_keep_capabilities(false) { - bail!(anyhow!(e).context("set keep capabilities returned")); - }; + prctl::set_keep_capabilities(false) + .map_err(|e| anyhow!(e).context("set keep capabilities returned"))?; + Ok(()) } @@ -1325,13 +1320,13 @@ impl LinuxContainer { // validate oci spec validator::validate(&config)?; - if let Err(e) = fs::create_dir_all(root.as_str()) { + fs::create_dir_all(root.as_str()).map_err(|e| { if e.kind() == std::io::ErrorKind::AlreadyExists { - return Err(e).context(format!("container {} already exists", id.as_str())); + return anyhow!(e).context(format!("container {} already exists", id.as_str())); } - return Err(e).context(format!("fail to create container directory {}", root)); - } + anyhow!(e).context(format!("fail to create container directory {}", root)) + })?; unistd::chown( root.as_str(), diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 0dc8667f84..45c73c1cf1 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -512,14 +512,12 @@ fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Re let args: Vec<&CStr> = vec![]; // run shell - if let Err(e) = unistd::execvp(cmd.as_c_str(), args.as_slice()) { - match e { - nix::Error::Sys(errno) => { - std::process::exit(errno as i32); - } - _ => std::process::exit(-2), + let _ = unistd::execvp(cmd.as_c_str(), args.as_slice()).map_err(|e| match e { + nix::Error::Sys(errno) => { + std::process::exit(errno as i32); } - } + _ => std::process::exit(-2), + }); } Ok(ForkResult::Parent { child: child_pid }) => { diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 8e988af146..45d89bd520 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -251,10 +251,7 @@ fn ephemeral_storage_handler( return Ok("".to_string()); } - if let Err(err) = fs::create_dir_all(Path::new(&storage.mount_point)) { - return Err(err.into()); - } - + fs::create_dir_all(Path::new(&storage.mount_point))?; common_storage_handler(logger, storage)?; Ok("".to_string()) diff --git a/src/agent/src/namespace.rs b/src/agent/src/namespace.rs index 892332b3d7..6cb42aa768 100644 --- a/src/agent/src/namespace.rs +++ b/src/agent/src/namespace.rs @@ -131,12 +131,12 @@ impl Namespace { }; let bare_mount = BareMount::new(source, destination, "none", flags, "", &logger); - if let Err(err) = bare_mount.mount() { - return Err(format!( + bare_mount.mount().map_err(|e| { + format!( "Failed to mount {} to {} with err:{:?}", - source, destination, err - )); - } + source, destination, e + ) + })?; Ok(()) }); diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index d7b95cd7b6..ed383b21ae 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -258,15 +258,12 @@ impl agentService { }); }); - if let Err(_) = rx.recv_timeout(Duration::from_secs(req.timeout as u64)) { - return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::ETIME))); - } + rx.recv_timeout(Duration::from_secs(req.timeout as u64)) + .map_err(|_| anyhow!(nix::Error::from_errno(nix::errno::Errno::ETIME)))?; - if let Err(_) = handle.join() { - return Err(anyhow!(nix::Error::from_errno( - nix::errno::Errno::UnknownErrno - ))); - } + handle + .join() + .map_err(|_| anyhow!(nix::Error::from_errno(nix::errno::Errno::UnknownErrno)))?; let s = self.sandbox.clone(); let mut sandbox = s.lock().unwrap(); @@ -903,12 +900,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { }; let err = libc::ioctl(fd, TIOCSWINSZ, &win); - if let Err(e) = Errno::result(err).map(drop) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( + Errno::result(err).map(drop).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e), - ))); - } + )) + })?; } Ok(Empty::new()) @@ -1062,12 +1059,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { s.running = true; if !req.guest_hook_path.is_empty() { - if let Err(e) = s.add_hooks(&req.guest_hook_path) { + let _ = s.add_hooks(&req.guest_hook_path).map_err(|e| { error!( sl!(), "add guest hook {} failed: {:?}", req.guest_hook_path, e ); - } + }); } if req.sandbox_id.len() > 0 { @@ -1168,12 +1165,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { let s = Arc::clone(&self.sandbox); let sandbox = s.lock().unwrap(); - if let Err(e) = sandbox.online_cpu_memory(&req) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + sandbox.online_cpu_memory(&req).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } @@ -1183,12 +1177,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::ReseedRandomDevRequest, ) -> ttrpc::Result { - if let Err(e) = random::reseed_rng(req.data.as_slice()) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + random::reseed_rng(req.data.as_slice()).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } @@ -1227,12 +1218,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::MemHotplugByProbeRequest, ) -> ttrpc::Result { - if let Err(e) = do_mem_hotplug_by_probe(&req.memHotplugProbeAddr) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + do_mem_hotplug_by_probe(&req.memHotplugProbeAddr).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } @@ -1242,12 +1230,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::SetGuestDateTimeRequest, ) -> ttrpc::Result { - if let Err(e) = do_set_guest_date_time(req.Sec, req.Usec) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + do_set_guest_date_time(req.Sec, req.Usec).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } @@ -1257,12 +1242,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::CopyFileRequest, ) -> ttrpc::Result { - if let Err(e) = do_copy_file(&req) { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))); - } + do_copy_file(&req).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + })?; Ok(Empty::new()) } diff --git a/src/agent/src/uevent.rs b/src/agent/src/uevent.rs index de79705ec4..35e8515633 100644 --- a/src/agent/src/uevent.rs +++ b/src/agent/src/uevent.rs @@ -99,14 +99,14 @@ impl Uevent { let online_path = format!("{}/{}/online", SYSFS_DIR, &self.devpath); // It's a memory hot-add event. if online_path.starts_with(SYSFS_MEMORY_ONLINE_PATH) { - if let Err(e) = online_device(online_path.as_ref()) { + let _ = online_device(online_path.as_ref()).map_err(|e| { error!( *logger, "failed to online device"; "device" => &self.devpath, "error" => format!("{}", e), - ); - } + ) + }); return; } } From 69269146837784bb3c311432cd747b43a30a7cfa Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 15:17:30 +0800 Subject: [PATCH 096/124] agent: refactor namespace::setup to optimize error handling - Replace the return value with anyhow::Result. - Remove if let Err. - Remove match. Signed-off-by: Tim Zhang --- src/agent/src/namespace.rs | 46 +++++++++++++------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/src/agent/src/namespace.rs b/src/agent/src/namespace.rs index 6cb42aa768..f5c6fa3b0a 100644 --- a/src/agent/src/namespace.rs +++ b/src/agent/src/namespace.rs @@ -3,15 +3,15 @@ // SPDX-License-Identifier: Apache-2.0 // +use anyhow::{anyhow, Result}; use nix::mount::MsFlags; use nix::sched::{unshare, CloneFlags}; use nix::unistd::{getpid, gettid}; use std::fmt; use std::fs; use std::fs::File; -use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; -use std::thread; +use std::thread::{self}; use crate::mount::{BareMount, FLAGS}; use slog::Logger; @@ -77,10 +77,8 @@ impl Namespace { // setup creates persistent namespace without switching to it. // Note, pid namespaces cannot be persisted. - pub fn setup(mut self) -> Result { - if let Err(err) = fs::create_dir_all(&self.persistent_ns_dir) { - return Err(err.to_string()); - } + pub fn setup(mut self) -> Result { + fs::create_dir_all(&self.persistent_ns_dir)?; let ns_path = PathBuf::from(&self.persistent_ns_dir); let ns_type = self.ns_type.clone(); @@ -88,33 +86,23 @@ impl Namespace { let new_ns_path = ns_path.join(&ns_type.get()); - if let Err(err) = File::create(new_ns_path.as_path()) { - return Err(err.to_string()); - } + File::create(new_ns_path.as_path())?; self.path = new_ns_path.clone().into_os_string().into_string().unwrap(); let hostname = self.hostname.clone(); - let new_thread = thread::spawn(move || { + let new_thread = thread::spawn(move || -> Result<()> { let origin_ns_path = get_current_thread_ns_path(&ns_type.get()); - let _origin_ns_fd = match File::open(Path::new(&origin_ns_path)) { - Err(err) => return Err(err.to_string()), - Ok(file) => file.as_raw_fd(), - }; + File::open(Path::new(&origin_ns_path))?; // Create a new netns on the current thread. let cf = ns_type.get_flags().clone(); - if let Err(err) = unshare(cf) { - return Err(err.to_string()); - } + unshare(cf)?; if ns_type == NamespaceType::UTS && hostname.is_some() { - match nix::unistd::sethostname(hostname.unwrap()) { - Err(err) => return Err(err.to_string()), - Ok(_) => (), - } + nix::unistd::sethostname(hostname.unwrap())?; } // Bind mount the new namespace from the current thread onto the mount point to persist it. let source: &str = origin_ns_path.as_str(); @@ -132,22 +120,20 @@ impl Namespace { let bare_mount = BareMount::new(source, destination, "none", flags, "", &logger); bare_mount.mount().map_err(|e| { - format!( + anyhow!( "Failed to mount {} to {} with err:{:?}", - source, destination, e + source, + destination, + e ) })?; Ok(()) }); - match new_thread.join() { - Ok(t) => match t { - Err(err) => return Err(err), - Ok(()) => (), - }, - Err(err) => return Err(format!("Failed to join thread {:?}!", err)), - } + new_thread + .join() + .map_err(|e| anyhow!("Failed to join thread {:?}!", e))??; Ok(self) } From 9def624c054d34052cf7c34444810ce15165f64f Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 14:35:22 +0800 Subject: [PATCH 097/124] agent: replace `if let Err` with `or_else` Fixes #934 Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 11 +++++++---- src/agent/src/mount.rs | 9 ++++++--- src/agent/src/rpc.rs | 8 +++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 91657e322a..4e80799a07 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -487,23 +487,26 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { } log_child!(cfd_log, "join namespace {:?}", s); - if let Err(e) = sched::setns(fd, s) { + sched::setns(fd, s).or_else(|e| { if s == CloneFlags::CLONE_NEWUSER { if e.as_errno().unwrap() != Errno::EINVAL { check!( write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), "write_sync for CLONE_NEWUSER" ); - return Err(e.into()); + return Err(e); } + + Ok(()) } else { check!( write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), "write_sync for sched::setns" ); - return Err(e.into()); + Err(e) } - } + })?; + unistd::close(fd)?; if s == CloneFlags::CLONE_NEWUSER { diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 45d89bd520..a9c2ba1b08 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -479,15 +479,18 @@ fn mount_to_rootfs(logger: &Logger, m: &INIT_MOUNT) -> Result<()> { fs::create_dir_all(Path::new(m.dest)).context("could not create directory")?; - if let Err(err) = bare_mount.mount() { + bare_mount.mount().or_else(|e| { if m.src != "dev" { - return Err(err.into()); + return Err(e); } + error!( logger, "Could not mount filesystem from {} to {}", m.src, m.dest ); - } + + Ok(()) + })?; Ok(()) } diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index ed383b21ae..fcfa4adcfa 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -1610,11 +1610,13 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> { PathBuf::from("/") }; - if let Err(e) = fs::create_dir_all(dir.to_str().unwrap()) { + fs::create_dir_all(dir.to_str().unwrap()).or_else(|e| { if e.kind() != std::io::ErrorKind::AlreadyExists { - return Err(e.into()); + return Err(e); } - } + + Ok(()) + })?; std::fs::set_permissions( dir.to_str().unwrap(), From 82ed34aee188951637b9ddc6ccf526d159db3443 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 15:26:35 +0800 Subject: [PATCH 098/124] agent: remove `check!` in child process because we cant' see logs. The check macro will log the errors but the log in child process can't be seen, just ignore it. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 33 ++++++++++------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 4e80799a07..a6eac8bceb 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -335,10 +335,7 @@ pub fn init_child() { Ok(_) => (), Err(e) => { log_child!(cfd_log, "child exit: {:?}", e); - check!( - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), - "write_sync in init_child()" - ); + let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); return; } } @@ -490,19 +487,13 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { sched::setns(fd, s).or_else(|e| { if s == CloneFlags::CLONE_NEWUSER { if e.as_errno().unwrap() != Errno::EINVAL { - check!( - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), - "write_sync for CLONE_NEWUSER" - ); + let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); return Err(e); } Ok(()) } else { - check!( - write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()), - "write_sync for sched::setns" - ); + let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); Err(e) } })?; @@ -578,14 +569,12 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { if guser.additional_gids.len() > 0 { setgroups(guser.additional_gids.as_slice()).map_err(|e| { - check!( - write_sync( - cwfd, - SYNC_FAILED, - format!("setgroups failed: {:?}", e).as_str() - ), - "write_sync for setgroups" + let _ = write_sync( + cwfd, + SYNC_FAILED, + format!("setgroups failed: {:?}", e).as_str(), ); + e })?; } @@ -650,9 +639,9 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { // notify parent that the child's ready to start write_sync(cwfd, SYNC_SUCCESS, "")?; log_child!(cfd_log, "ready to run exec"); - check!(unistd::close(cfd_log), "closing cfd log"); - check!(unistd::close(crfd), "closing crfd"); - check!(unistd::close(cwfd), "closing cwfd"); + let _ = unistd::close(cfd_log); + let _ = unistd::close(crfd); + let _ = unistd::close(cwfd); if oci_process.terminal { unistd::setsid()?; From 7c0d68f7f7ee0c4800be128f7912043e875dab7f Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 15:49:09 +0800 Subject: [PATCH 099/124] agent: replace check! with map_err for readability It's ambiguous and not easy to read to call method use macro. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 76 +++++++++++------------------ 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index a6eac8bceb..d65d6b4279 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -67,17 +67,6 @@ const CLOG_FD: &str = "CLOG_FD"; const FIFO_FD: &str = "FIFO_FD"; const HOME_ENV_KEY: &str = "HOME"; -#[macro_export] -macro_rules! check { - ($what:expr, $where:expr) => ({ - if let Err(e) = $what { - let subsystem = $where; - let logger = slog_scope::logger().new(o!("subsystem" => subsystem)); - warn!(logger, "{:?}", e); - } - }) -} - #[derive(PartialEq, Clone, Copy)] pub enum Status { CREATED, @@ -778,10 +767,9 @@ impl BaseContainer for LinuxContainer { let st = self.oci_state()?; let (pfd_log, cfd_log) = unistd::pipe().context("failed to create pipe")?; - check!( - fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), - "fcntl pfd log FD_CLOEXEC" - ); + + let _ = fcntl::fcntl(pfd_log, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) + .map_err(|e| warn!(logger, "fcntl pfd log FD_CLOEXEC {:?}", e)); let child_logger = logger.new(o!("action" => "child process log")); let log_handler = thread::spawn(move || { @@ -810,18 +798,16 @@ impl BaseContainer for LinuxContainer { info!(logger, "exec fifo opened!"); let (prfd, cwfd) = unistd::pipe().context("failed to create pipe")?; let (crfd, pwfd) = unistd::pipe().context("failed to create pipe")?; - check!( - fcntl::fcntl(prfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), - "fcntl prfd FD_CLOEXEC" - ); - check!( - fcntl::fcntl(pwfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), - "fcntl pwfd FD_COLEXEC" - ); + + let _ = fcntl::fcntl(prfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) + .map_err(|e| warn!(logger, "fcntl prfd FD_CLOEXEC {:?}", e)); + + let _ = fcntl::fcntl(pwfd, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) + .map_err(|e| warn!(logger, "fcntl pwfd FD_COLEXEC {:?}", e)); defer!({ - check!(unistd::close(prfd), "close prfd"); - check!(unistd::close(pwfd), "close pwfd"); + let _ = unistd::close(prfd).map_err(|e| warn!(logger, "close prfd {:?}", e)); + let _ = unistd::close(pwfd).map_err(|e| warn!(logger, "close pwfd {:?}", e)); }); let child_stdin: std::process::Stdio; @@ -831,14 +817,10 @@ impl BaseContainer for LinuxContainer { if tty { let pseudo = pty::openpty(None, None)?; p.term_master = Some(pseudo.master); - check!( - fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), - "fnctl pseudo.master" - ); - check!( - fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)), - "fcntl pseudo.slave" - ); + let _ = fcntl::fcntl(pseudo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) + .map_err(|e| warn!(logger, "fnctl pseudo.master {:?}", e)); + let _ = fcntl::fcntl(pseudo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)) + .map_err(|e| warn!(logger, "fcntl pseudo.slave {:?}", e)); child_stdin = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; child_stdout = unsafe { std::process::Stdio::from_raw_fd(pseudo.slave) }; @@ -865,11 +847,10 @@ impl BaseContainer for LinuxContainer { //restore the parent's process's pid namespace. defer!({ - check!( - sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID), - "settns CLONE_NEWPID" - ); - check!(unistd::close(old_pid_ns), "close old pid namespace"); + let _ = sched::setns(old_pid_ns, CloneFlags::CLONE_NEWPID) + .map_err(|e| warn!(logger, "settns CLONE_NEWPID {:?}", e)); + let _ = unistd::close(old_pid_ns) + .map_err(|e| warn!(logger, "close old pid namespace {:?}", e)); }); let pidns = get_pid_namespace(&self.logger, linux)?; @@ -911,7 +892,7 @@ impl BaseContainer for LinuxContainer { } if p.init { - check!(unistd::close(fifofd), "close fifofd"); + let _ = unistd::close(fifofd).map_err(|e| warn!(logger, "close fifofd {:?}", e)); } info!(logger, "child pid: {}", p.pid); @@ -929,10 +910,8 @@ impl BaseContainer for LinuxContainer { Err(e) => { error!(logger, "create container process error {:?}", e); // kill the child process. - check!( - signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)), - "signal::kill joining namespaces" - ); + let _ = signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)) + .map_err(|e| warn!(logger, "signal::kill joining namespaces {:?}", e)); return Err(e); } }; @@ -945,10 +924,9 @@ impl BaseContainer for LinuxContainer { let (exit_pipe_r, exit_pipe_w) = unistd::pipe2(OFlag::O_CLOEXEC) .context("failed to create pipe") .map_err(|e| { - check!( - signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL)), - "signal::kill creating pipe" - ); + let _ = signal::kill(Pid::from_raw(child.id() as i32), Some(Signal::SIGKILL)) + .map_err(|e| warn!(logger, "signal::kill creating pipe {:?}", e)); + e })?; @@ -962,7 +940,9 @@ impl BaseContainer for LinuxContainer { self.processes.insert(p.pid, p); info!(logger, "wait on child log handler"); - check!(log_handler.join(), "joining log handler"); + let _ = log_handler + .join() + .map_err(|e| warn!(logger, "joining log handler {:?}", e)); info!(logger, "create process completed"); return Ok(()); } From 64e4b2fa8337749311dd5ef503d2a410fe2d22b4 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 17:54:10 +0800 Subject: [PATCH 100/124] agent: replace unnecessary `match Result` with `map_err` Replace `match Result` whose Ok hand is useless. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 76 +++++++++------------ src/agent/rustjail/src/mount.rs | 100 ++++++++++++---------------- src/agent/src/device.rs | 19 +++--- src/agent/src/rpc.rs | 91 +++++++++---------------- src/agent/src/sandbox.rs | 7 +- 5 files changed, 117 insertions(+), 176 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index d65d6b4279..c96f79468b 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -320,14 +320,11 @@ impl Container for LinuxContainer { pub fn init_child() { let cwfd = std::env::var(CWFD_FD).unwrap().parse::().unwrap(); let cfd_log = std::env::var(CLOG_FD).unwrap().parse::().unwrap(); - match do_init_child(cwfd) { - Ok(_) => (), - Err(e) => { - log_child!(cfd_log, "child exit: {:?}", e); - let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); - return; - } - } + + let _ = do_init_child(cwfd).map_err(|e| { + log_child!(cfd_log, "child exit: {:?}", e); + let _ = write_sync(cwfd, SYNC_FAILED, format!("{:?}", e).as_str()); + }); } fn do_init_child(cwfd: RawFd) -> Result<()> { @@ -392,9 +389,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { to_new.set(*s, true); } } else { - let fd = match fcntl::open(ns.path.as_str(), OFlag::O_CLOEXEC, Mode::empty()) { - Ok(v) => v, - Err(e) => { + let fd = + fcntl::open(ns.path.as_str(), OFlag::O_CLOEXEC, Mode::empty()).map_err(|e| { log_child!( cfd_log, "cannot open type: {} path: {}", @@ -402,9 +398,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { ns.path.clone() ); log_child!(cfd_log, "error is : {}", e.as_errno().unwrap().desc()); - return Err(e.into()); - } - }; + e + })?; if *s != CloneFlags::CLONE_NEWPID { to_join.push((*s, fd)); @@ -834,16 +829,14 @@ impl BaseContainer for LinuxContainer { child_stderr = unsafe { std::process::Stdio::from_raw_fd(stderr) }; } - let old_pid_ns = match fcntl::open(PID_NS_PATH, OFlag::O_CLOEXEC, Mode::empty()) { - Ok(v) => v, - Err(e) => { + let old_pid_ns = + fcntl::open(PID_NS_PATH, OFlag::O_CLOEXEC, Mode::empty()).map_err(|e| { error!( logger, "cannot open pid ns path: {} with error: {:?}", PID_NS_PATH, e ); - return Err(e.into()); - } - }; + e + })?; //restore the parent's process's pid namespace. defer!({ @@ -897,7 +890,7 @@ impl BaseContainer for LinuxContainer { info!(logger, "child pid: {}", p.pid); - match join_namespaces( + join_namespaces( &logger, &spec, &p, @@ -905,16 +898,15 @@ impl BaseContainer for LinuxContainer { &st, pwfd, prfd, - ) { - Ok(_) => (), - Err(e) => { - error!(logger, "create container process error {:?}", e); - // kill the child process. - let _ = signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)) - .map_err(|e| warn!(logger, "signal::kill joining namespaces {:?}", e)); - return Err(e); - } - }; + ) + .map_err(|e| { + error!(logger, "create container process error {:?}", e); + // kill the child process. + let _ = signal::kill(Pid::from_raw(p.pid), Some(Signal::SIGKILL)) + .map_err(|e| warn!(logger, "signal::kill joining namespaces {:?}", e)); + + e + })?; info!(logger, "entered namespaces!"); @@ -1085,9 +1077,8 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result> { return Ok(None); } - let fd = match fcntl::open(ns.path.as_str(), OFlag::O_CLOEXEC, Mode::empty()) { - Ok(v) => v, - Err(e) => { + let fd = + fcntl::open(ns.path.as_str(), OFlag::O_CLOEXEC, Mode::empty()).map_err(|e| { error!( logger, "cannot open type: {} path: {}", @@ -1095,9 +1086,9 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result> { ns.path.clone() ); error!(logger, "error is : {}", e.as_errno().unwrap().desc()); - return Err(e.into()); - } - }; + + e + })?; return Ok(Some(fd)); } @@ -1245,13 +1236,10 @@ fn write_mappings(logger: &Logger, path: &str, maps: &[LinuxIDMapping]) -> Resul if !data.is_empty() { let fd = fcntl::open(path, OFlag::O_WRONLY, Mode::empty())?; defer!(unistd::close(fd).unwrap()); - match unistd::write(fd, data.as_bytes()) { - Ok(_) => {} - Err(e) => { - info!(logger, "cannot write mapping"); - return Err(e.into()); - } - } + unistd::write(fd, data.as_bytes()).map_err(|e| { + info!(logger, "cannot write mapping"); + e + })?; } Ok(()) } diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index f759456e7a..daa19ce0b0 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -407,20 +407,17 @@ fn mount_cgroups( if key != base { let src = format!("{}/{}", m.destination.as_str(), key); - match unix::fs::symlink(destination.as_str(), &src[1..]) { - Err(e) => { - log_child!( - cfd_log, - "symlink: {} {} err: {}", - key, - destination.as_str(), - e.to_string() - ); + unix::fs::symlink(destination.as_str(), &src[1..]).map_err(|e| { + log_child!( + cfd_log, + "symlink: {} {} err: {}", + key, + destination.as_str(), + e.to_string() + ); - return Err(e.into()); - } - Ok(_) => {} - } + e + })?; } } @@ -689,18 +686,14 @@ fn mount_from( Path::new(&dest) }; - // let _ = fs::create_dir_all(&dir); - match fs::create_dir_all(&dir) { - Ok(_) => {} - Err(e) => { - log_child!( - cfd_log, - "creat dir {}: {}", - dir.to_str().unwrap(), - e.to_string() - ); - } - } + let _ = fs::create_dir_all(&dir).map_err(|e| { + log_child!( + cfd_log, + "creat dir {}: {}", + dir.to_str().unwrap(), + e.to_string() + ) + }); // make sure file exists so we can bind over it if src.is_file() { @@ -717,31 +710,26 @@ fn mount_from( } }; - match stat::stat(dest.as_str()) { - Ok(_) => {} - Err(e) => { - log_child!( - cfd_log, - "dest stat error. {}: {}", - dest.as_str(), - e.as_errno().unwrap().desc() - ); - } - } + let _ = stat::stat(dest.as_str()).map_err(|e| { + log_child!( + cfd_log, + "dest stat error. {}: {}", + dest.as_str(), + e.as_errno().unwrap().desc() + ) + }); - match mount( + mount( Some(src.as_str()), dest.as_str(), Some(m.r#type.as_str()), flags, Some(d.as_str()), - ) { - Ok(_) => {} - Err(e) => { - log_child!(cfd_log, "mount error: {}", e.as_errno().unwrap().desc()); - return Err(e.into()); - } - } + ) + .map_err(|e| { + log_child!(cfd_log, "mount error: {}", e.as_errno().unwrap().desc()); + e + })?; if flags.contains(MsFlags::MS_BIND) && flags.intersects( @@ -753,24 +741,22 @@ fn mount_from( | MsFlags::MS_SLAVE), ) { - match mount( + mount( Some(dest.as_str()), dest.as_str(), None::<&str>, flags | MsFlags::MS_REMOUNT, None::<&str>, - ) { - Err(e) => { - log_child!( - cfd_log, - "remout {}: {}", - dest.as_str(), - e.as_errno().unwrap().desc() - ); - return Err(e.into()); - } - Ok(_) => {} - } + ) + .map_err(|e| { + log_child!( + cfd_log, + "remout {}: {}", + dest.as_str(), + e.as_errno().unwrap().desc() + ); + e + })?; } Ok(()) } diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 18f53e0c9e..2a128d8388 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -137,17 +137,14 @@ fn get_device_name(sandbox: &Arc>, dev_addr: &str) -> Result name, - Err(_) => { - GLOBAL_DEVICE_WATCHER.lock().unwrap().remove_entry(dev_addr); - return Err(anyhow!( - "Timeout reached after {:?} waiting for device {}", - hotplug_timeout, - dev_addr - )); - } - }; + let dev_name = rx.recv_timeout(hotplug_timeout).map_err(|_| { + GLOBAL_DEVICE_WATCHER.lock().unwrap().remove_entry(dev_addr); + anyhow!( + "Timeout reached after {:?} waiting for device {}", + hotplug_timeout, + dev_addr + ) + })?; Ok(format!("{}/{}", SYSTEM_DEV_PATH, &dev_name)) } diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index fcfa4adcfa..7719608994 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -588,13 +588,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::WaitProcessRequest, ) -> ttrpc::Result { - match self.do_wait_process(req) { - Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))), - Ok(resp) => Ok(resp), - } + self.do_wait_process(req).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + }) } fn list_processes( @@ -734,13 +730,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { "invalid container id".to_string(), )))?; - match ctr.stats() { - Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))), - Ok(resp) => Ok(resp), - } + ctr.stats().map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + }) } fn pause_container( @@ -794,13 +786,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::WriteStreamRequest, ) -> ttrpc::Result { - match self.do_write_stream(req) { - Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))), - Ok(resp) => Ok(resp), - } + self.do_write_stream(req).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + }) } fn read_stdout( @@ -808,13 +796,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::ReadStreamRequest, ) -> ttrpc::Result { - match self.do_read_stream(req, true) { - Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))), - Ok(resp) => Ok(resp), - } + self.do_read_stream(req, true).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + }) } fn read_stderr( @@ -822,13 +806,9 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::ReadStreamRequest, ) -> ttrpc::Result { - match self.do_read_stream(req, false) { - Err(e) => Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INTERNAL, - e.to_string(), - ))), - Ok(resp) => Ok(resp), - } + self.do_read_stream(req, false).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status(ttrpc::Code::INTERNAL, e.to_string())) + }) } fn close_stdin( @@ -841,15 +821,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let p = match find_process(&mut sandbox, cid.as_str(), eid.as_str(), false) { - Ok(v) => v, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::INVALID_ARGUMENT, - format!("invalid argument: {:?}", e), - ))); - } - }; + let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + format!("invalid argument: {:?}", e), + )) + })?; if p.term_master.is_some() { let _ = unistd::close(p.term_master.unwrap()); @@ -873,15 +850,12 @@ impl protocols::agent_ttrpc::AgentService for agentService { let eid = req.exec_id.clone(); let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); - let p = match find_process(&mut sandbox, cid.as_str(), eid.as_str(), false) { - Ok(v) => v, - Err(e) => { - return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( - ttrpc::Code::UNAVAILABLE, - format!("invalid argument: {:?}", e), - ))); - } - }; + let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| { + ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::UNAVAILABLE, + format!("invalid argument: {:?}", e), + )) + })?; if p.term_master.is_none() { return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( @@ -1335,13 +1309,10 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { return Err(anyhow!("Invalid block size")); } - size = match u64::from_str_radix(v.trim(), 16) { - Ok(h) => h, - Err(_) => { - warn!(sl!(), "failed to parse the str {} to hex", size); - return Err(anyhow!("Invalid block size")); - } - }; + size = u64::from_str_radix(v.trim(), 16).map_err(|_| { + warn!(sl!(), "failed to parse the str {} to hex", size); + anyhow!("Invalid block size") + })?; } Err(e) => { info!(sl!(), "memory block size error: {:?}", e.kind()); diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index f871d3ef72..f2d92e9728 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -316,10 +316,9 @@ impl Sandbox { thread::spawn(move || { for event in rx { info!(logger, "got an OOM event {:?}", event); - match tx.send(container_id.clone()) { - Err(err) => error!(logger, "failed to send message: {:?}", err), - Ok(_) => {} - } + let _ = tx + .send(container_id.clone()) + .map_err(|e| error!(logger, "failed to send message: {:?}", e)); } }); } From 8f8061da08fac24ef5880f561f1a1fe59d76c82d Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 19:01:41 +0800 Subject: [PATCH 101/124] agent: replace `match Result` with `or_else` `or_else` is suitable for more complicated situations. We can use it to return Ok in Err handling. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/mount.rs | 33 ++++++++++++++++----------------- src/agent/rustjail/src/sync.rs | 22 ++++++++-------------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index daa19ce0b0..cc3b928a4a 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -612,24 +612,23 @@ pub fn ms_move_root(rootfs: &str) -> Result { MsFlags::MS_SLAVE | MsFlags::MS_REC, None::<&str>, )?; - match umount2(abs_mount_point, MntFlags::MNT_DETACH) { - Ok(_) => (), - Err(e) => { - if e.ne(&nix::Error::from(Errno::EINVAL)) && e.ne(&nix::Error::from(Errno::EPERM)) { - return Err(anyhow!(e)); - } - - // If we have not privileges for umounting (e.g. rootless), then - // cover the path. - mount( - Some("tmpfs"), - abs_mount_point, - Some("tmpfs"), - MsFlags::empty(), - None::<&str>, - )?; + umount2(abs_mount_point, MntFlags::MNT_DETACH).or_else(|e| { + if e.ne(&nix::Error::from(Errno::EINVAL)) && e.ne(&nix::Error::from(Errno::EPERM)) { + return Err(anyhow!(e)); } - } + + // If we have not privileges for umounting (e.g. rootless), then + // cover the path. + mount( + Some("tmpfs"), + abs_mount_point, + Some("tmpfs"), + MsFlags::empty(), + None::<&str>, + )?; + + Ok(()) + })?; } mount( diff --git a/src/agent/rustjail/src/sync.rs b/src/agent/rustjail/src/sync.rs index 8ce43b270f..9e98b0ad76 100644 --- a/src/agent/rustjail/src/sync.rs +++ b/src/agent/rustjail/src/sync.rs @@ -143,21 +143,15 @@ pub fn write_sync(fd: RawFd, msg_type: i32, data_str: &str) -> Result<()> { }, SYNC_DATA => { let length: i32 = data_str.len() as i32; - match write_count(fd, &length.to_be_bytes(), MSG_SIZE) { - Ok(_count) => (), - Err(e) => { - unistd::close(fd)?; - return Err(anyhow!(e).context("error in send message to process")); - } - } + write_count(fd, &length.to_be_bytes(), MSG_SIZE).or_else(|e| { + unistd::close(fd)?; + Err(anyhow!(e).context("error in send message to process")) + })?; - match write_count(fd, data_str.as_bytes(), data_str.len()) { - Ok(_count) => (), - Err(e) => { - unistd::close(fd)?; - return Err(anyhow!(e).context("error in send message to process")); - } - } + write_count(fd, data_str.as_bytes(), data_str.len()).or_else(|e| { + unistd::close(fd)?; + Err(anyhow!(e).context("error in send message to process")) + })?; } _ => (), From 6e4da19fa5216052111f359400f47eb92ce2f328 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 10:00:54 +0100 Subject: [PATCH 102/124] agent: Fix crasher if UpdateInterface request empty Check if the interface specified in the `UpdateInterface` API is set before using it to avoid crashing the agent. Fixes: #950. Signed-off-by: James O. D. Hunt --- src/agent/src/rpc.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 7719608994..bc09f8e484 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -890,6 +890,13 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::UpdateInterfaceRequest, ) -> ttrpc::Result { + if req.interface.is_none() { + return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + format!("empty update interface request"), + ))); + } + let interface = req.interface.clone(); let s = Arc::clone(&self.sandbox); let mut sandbox = s.lock().unwrap(); @@ -1710,7 +1717,27 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use crate::protocols::agent_ttrpc::AgentService; use oci::{Hook, Hooks}; + use std::sync::mpsc::{Receiver, Sender}; + use ttrpc::{MessageHeader, TtrpcContext}; + + fn mk_ttrpc_context() -> (TtrpcContext, Receiver<(MessageHeader, Vec)>) { + let mh = MessageHeader::default(); + + let (tx, rx): ( + Sender<(MessageHeader, Vec)>, + Receiver<(MessageHeader, Vec)>, + ) = channel(); + + let ctx = TtrpcContext { + fd: -1, + mh: mh, + res_tx: tx, + }; + + (ctx, rx) + } #[test] fn test_load_kernel_module() { @@ -1750,4 +1777,21 @@ mod tests { append_guest_hooks(&s, &mut oci); assert_eq!(s.hooks, oci.hooks); } + + #[test] + fn test_update_interface() { + let logger = slog::Logger::root(slog::Discard, o!()); + let sandbox = Sandbox::new(&logger).unwrap(); + + let agent_service = Box::new(agentService { + sandbox: Arc::new(Mutex::new(sandbox)), + }); + + let req = protocols::agent::UpdateInterfaceRequest::default(); + let (ctx, _) = mk_ttrpc_context(); + + let result = agent_service.update_interface(&ctx, req); + + assert!(result.is_err(), "expected update interface to fail"); + } } From 76408c0f13f4318b3ebcc2d97b326601d6334eea Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 10:03:13 +0100 Subject: [PATCH 103/124] agent: Fix crasher if UpdateRoutes request empty Check if the routes specified in the `UpdateRoutes` API is set before using it to avoid crashing the agent. Fixes: #949. Signed-off-by: James O. D. Hunt --- src/agent/src/rpc.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index bc09f8e484..cbb20469bd 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -925,6 +925,13 @@ impl protocols::agent_ttrpc::AgentService for agentService { req: protocols::agent::UpdateRoutesRequest, ) -> ttrpc::Result { let mut routes = protocols::agent::Routes::new(); + if req.routes.is_none() { + return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + format!("empty update routes request"), + ))); + } + let rs = req.routes.clone().unwrap().Routes.into_vec(); let s = Arc::clone(&self.sandbox); @@ -1794,4 +1801,21 @@ mod tests { assert!(result.is_err(), "expected update interface to fail"); } + + #[test] + fn test_update_routes() { + let logger = slog::Logger::root(slog::Discard, o!()); + let sandbox = Sandbox::new(&logger).unwrap(); + + let agent_service = Box::new(agentService { + sandbox: Arc::new(Mutex::new(sandbox)), + }); + + let req = protocols::agent::UpdateRoutesRequest::default(); + let (ctx, _) = mk_ttrpc_context(); + + let result = agent_service.update_routes(&ctx, req); + + assert!(result.is_err(), "expected update routes to fail"); + } } From d413bf7d444507bf0807d98ad6291e24b6ea87a4 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 10:05:27 +0100 Subject: [PATCH 104/124] agent: Fix crasher if AddARPNeighbors request empty Check if the ARP neighbours specified in the `AddARPNeighbors` API is set before using it to avoid crashing the agent. Fixes: #955. Signed-off-by: James O. D. Hunt --- src/agent/src/rpc.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index cbb20469bd..ac9b5c5c3b 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -1127,6 +1127,13 @@ impl protocols::agent_ttrpc::AgentService for agentService { _ctx: &ttrpc::TtrpcContext, req: protocols::agent::AddARPNeighborsRequest, ) -> ttrpc::Result { + if req.neighbors.is_none() { + return Err(ttrpc::Error::RpcStatus(ttrpc::get_status( + ttrpc::Code::INVALID_ARGUMENT, + format!("empty add arp neighbours request"), + ))); + } + let neighs = req.neighbors.clone().unwrap().ARPNeighbors.into_vec(); let s = Arc::clone(&self.sandbox); @@ -1818,4 +1825,21 @@ mod tests { assert!(result.is_err(), "expected update routes to fail"); } + + #[test] + fn test_add_arp_neighbors() { + let logger = slog::Logger::root(slog::Discard, o!()); + let sandbox = Sandbox::new(&logger).unwrap(); + + let agent_service = Box::new(agentService { + sandbox: Arc::new(Mutex::new(sandbox)), + }); + + let req = protocols::agent::AddARPNeighborsRequest::default(); + let (ctx, _) = mk_ttrpc_context(); + + let result = agent_service.add_arp_neighbors(&ctx, req); + + assert!(result.is_err(), "expected add arp neighbors to fail"); + } } From bd816dfcec17bc2aa6baa5a9fdd438352733f602 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 19:14:43 +0800 Subject: [PATCH 105/124] agent: Use `ok_or_else` instead of match for Option -> Result Using ok_or is clearer than match. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 15 +++++---------- src/agent/src/device.rs | 16 ++++++++-------- src/agent/src/mount.rs | 20 +++++++++----------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index c96f79468b..9add8d152e 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -612,12 +612,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { let exec_file = Path::new(&args[0]); log_child!(cfd_log, "process command: {:?}", &args); if !exec_file.exists() { - match find_file(exec_file) { - Some(_) => (), - None => { - return Err(anyhow!("the file {} is not exist", &args[0])); - } - } + find_file(exec_file).ok_or_else(|| anyhow!("the file {} is not exist", &args[0]))?; } // notify parent that the child's ready to start @@ -1047,10 +1042,10 @@ fn do_exec(args: &[String]) -> ! { fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> Result<()> { info!(logger, "updating namespaces"); - let linux = match spec.linux.as_mut() { - None => return Err(anyhow!("Spec didn't contain linux field")), - Some(l) => l, - }; + let linux = spec + .linux + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?; let namespaces = linux.namespaces.as_mut_slice(); for namespace in namespaces.iter_mut() { diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 2a128d8388..0cd2b1d72c 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -211,10 +211,10 @@ fn update_spec_device_list(device: &Device, spec: &mut Spec, devidx: &DevIndex) )); } - let linux = match spec.linux.as_mut() { - None => return Err(anyhow!("Spec didn't container linux field")), - Some(l) => l, - }; + let linux = spec + .linux + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; if !Path::new(&device.vm_path).exists() { return Err(anyhow!("vm_path:{} doesn't exist", device.vm_path)); @@ -408,10 +408,10 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> { let major = stat::major(rdev) as i64; let minor = stat::minor(rdev) as i64; - let linux = match spec.linux.as_mut() { - None => return Err(anyhow!("Spec didn't container linux field")), - Some(l) => l, - }; + let linux = spec + .linux + .as_mut() + .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; if linux.resources.is_none() { linux.resources = Some(LinuxResources::default()); diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index a9c2ba1b08..a44956c04d 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -446,15 +446,14 @@ pub fn add_storages( "subsystem" => "storage", "storage-type" => handler_name.to_owned())); - let handler = match STORAGEHANDLERLIST.get(&handler_name.as_str()) { - None => { - return Err(anyhow!( + let handler = STORAGEHANDLERLIST + .get(&handler_name.as_str()) + .ok_or_else(|| { + anyhow!( "Failed to find the storage handler {}", storage.driver.to_owned() - )); - } - Some(f) => f, - }; + ) + })?; let mount_point = match handler(&logger, &storage, sandbox.clone()) { // Todo need to rollback the mounted storage if err met. @@ -659,10 +658,9 @@ pub fn remove_mounts(mounts: &Vec) -> Result<()> { fn ensure_destination_exists(destination: &str, fs_type: &str) -> Result<()> { let d = Path::new(destination); if !d.exists() { - let dir = match d.parent() { - Some(d) => d, - None => return Err(anyhow!("mount destination {} doesn't exist", destination)), - }; + let dir = d + .parent() + .ok_or_else(|| anyhow!("mount destination {} doesn't exist", destination))?; if !dir.exists() { fs::create_dir_all(dir).context(format!("create dir all failed on {:?}", dir))?; } From 095d4ad08d01f1059e4a54fde12460e2c1dcb1b3 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 19:19:51 +0800 Subject: [PATCH 106/124] agent: remove useless match Remove useless match. Signed-off-by: Tim Zhang --- src/agent/src/mount.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index a44956c04d..c31afab478 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -455,11 +455,8 @@ pub fn add_storages( ) })?; - let mount_point = match handler(&logger, &storage, sandbox.clone()) { - // Todo need to rollback the mounted storage if err met. - Err(e) => return Err(e), - Ok(m) => m, - }; + // Todo need to rollback the mounted storage if err met. + let mount_point = handler(&logger, &storage, sandbox.clone())?; if mount_point.len() > 0 { mount_list.push(mount_point); From ae2d89e95ed780ba562ae5d047c7ef83a7274c35 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 19:24:22 +0800 Subject: [PATCH 107/124] agent: use anyhow `context` to attach context to `Error` instead of `match` Context is clearer than match for these situations. Signed-off-by: Tim Zhang --- src/agent/src/sandbox.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index f2d92e9728..ac80f5be0f 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -166,23 +166,17 @@ impl Sandbox { pub fn setup_shared_namespaces(&mut self) -> Result { // Set up shared IPC namespace - self.shared_ipcns = match Namespace::new(&self.logger).as_ipc().setup() { - Ok(ns) => ns, - Err(err) => { - return Err(anyhow!(err).context("Failed to setup persistent IPC namespace")); - } - }; + self.shared_ipcns = Namespace::new(&self.logger) + .as_ipc() + .setup() + .context("Failed to setup persistent IPC namespace")?; // // Set up shared UTS namespace - self.shared_utsns = match Namespace::new(&self.logger) + self.shared_utsns = Namespace::new(&self.logger) .as_uts(self.hostname.as_str()) .setup() - { - Ok(ns) => ns, - Err(err) => { - return Err(anyhow!(err).context("Failed to setup persistent UTS namespace")); - } - }; + .context("Failed to setup persistent UTS namespace")?; + Ok(true) } From d658129695ce20d446923585e9988140e17d11ee Mon Sep 17 00:00:00 2001 From: bin liu Date: Thu, 15 Oct 2020 15:05:24 +0800 Subject: [PATCH 108/124] kata-monitor: use regexp to check if runtime is kata containers To support a few common configurations for Kata, including: - `io.containerd.kata.v2` - `io.containerd.kata-qemu.v2` - `io.containerd.kata-clh.v2` `kata-monintor` changes to use regexp instead of direct string comparison. Fixes: #957 Signed-off-by: bin liu --- .../cli/containerd-shim-kata-v2/main.go | 4 +-- src/runtime/pkg/kata-monitor/containerd.go | 2 +- src/runtime/pkg/kata-monitor/sandbox_cache.go | 8 ++++- src/runtime/pkg/types/types.go | 5 ++-- src/runtime/pkg/types/types_test.go | 30 +++++++++++++++++++ 5 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 src/runtime/pkg/types/types_test.go diff --git a/src/runtime/cli/containerd-shim-kata-v2/main.go b/src/runtime/cli/containerd-shim-kata-v2/main.go index 89f2589657..ed35338fa2 100644 --- a/src/runtime/cli/containerd-shim-kata-v2/main.go +++ b/src/runtime/cli/containerd-shim-kata-v2/main.go @@ -22,9 +22,9 @@ func shimConfig(config *shim.Config) { func main() { if len(os.Args) == 2 && os.Args[1] == "--version" { - fmt.Printf("%s containerd shim: id: %q, version: %s, commit: %v\n", project, types.KataRuntimeName, version, commit) + fmt.Printf("%s containerd shim: id: %q, version: %s, commit: %v\n", project, types.DefaultKataRuntimeName, version, commit) os.Exit(0) } - shim.Run(types.KataRuntimeName, containerdshim.New, shimConfig) + shim.Run(types.DefaultKataRuntimeName, containerdshim.New, shimConfig) } diff --git a/src/runtime/pkg/kata-monitor/containerd.go b/src/runtime/pkg/kata-monitor/containerd.go index a153dc544f..774cb98afe 100644 --- a/src/runtime/pkg/kata-monitor/containerd.go +++ b/src/runtime/pkg/kata-monitor/containerd.go @@ -82,7 +82,7 @@ func (ka *KataMonitor) getSandboxes() (map[string]string, error) { namespacedCtx := namespaces.WithNamespace(ctx, namespace) // only list Kata Containers pods/containers containers, err := client.ContainerService().List(namespacedCtx, - "runtime.name=="+types.KataRuntimeName+`,labels."io.cri-containerd.kind"==sandbox`) + "runtime.name~="+types.KataRuntimeNameRegexp+`,labels."io.cri-containerd.kind"==sandbox`) if err != nil { return err } diff --git a/src/runtime/pkg/kata-monitor/sandbox_cache.go b/src/runtime/pkg/kata-monitor/sandbox_cache.go index 974bedf3f4..8d3b579751 100644 --- a/src/runtime/pkg/kata-monitor/sandbox_cache.go +++ b/src/runtime/pkg/kata-monitor/sandbox_cache.go @@ -8,6 +8,7 @@ package katamonitor import ( "context" "fmt" + "regexp" "sync" "github.com/containerd/containerd" @@ -97,6 +98,11 @@ func (sc *sandboxCache) startEventsListener(addr string) error { `topic=="/containers/delete"`, } + runtimeNameRegexp, err := regexp.Compile(types.KataRuntimeNameRegexp) + if err != nil { + return err + } + eventsCh, errCh := eventsClient.Subscribe(ctx, eventFilters...) for { var e *events.Envelope @@ -138,7 +144,7 @@ func (sc *sandboxCache) startEventsListener(addr string) error { } // skip non-kata contaienrs - if cc.Runtime.Name != types.KataRuntimeName { + if !runtimeNameRegexp.MatchString(cc.Runtime.Name) { continue } diff --git a/src/runtime/pkg/types/types.go b/src/runtime/pkg/types/types.go index 66f48994d6..73a255117e 100644 --- a/src/runtime/pkg/types/types.go +++ b/src/runtime/pkg/types/types.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Ant Financial +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -6,6 +6,7 @@ package types const ( - KataRuntimeName = "io.containerd.kata.v2" + DefaultKataRuntimeName = "io.containerd.kata.v2" + KataRuntimeNameRegexp = `io\.containerd\.kata.*\.v2` ContainerdRuntimeTaskPath = "io.containerd.runtime.v2.task" ) diff --git a/src/runtime/pkg/types/types_test.go b/src/runtime/pkg/types/types_test.go new file mode 100644 index 0000000000..6da2c9bcea --- /dev/null +++ b/src/runtime/pkg/types/types_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2020 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// +package types + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKataRuntimeNameRegexp(t *testing.T) { + assert := assert.New(t) + + runtimeNameRegexp, err := regexp.Compile(KataRuntimeNameRegexp) + assert.NoError(err) + + // valid Kata containers name + assert.Equal(true, runtimeNameRegexp.MatchString("io.containerd.kata.v2")) + assert.Equal(true, runtimeNameRegexp.MatchString("io.containerd.kataclh.v2")) + assert.Equal(true, runtimeNameRegexp.MatchString("io.containerd.kata-clh.v2")) + assert.Equal(true, runtimeNameRegexp.MatchString("io.containerd.kata.1.2.3-clh.4.v2")) + + // invalid Kata containers name + assert.Equal(false, runtimeNameRegexp.MatchString("io2containerd.kata.v2")) + assert.Equal(false, runtimeNameRegexp.MatchString("io.c3ontainerd.kata.v2")) + assert.Equal(false, runtimeNameRegexp.MatchString("io.containerd.runc.v1")) +} From 05e9fe0591ffef5239ba164a924443ab5a6bc9ae Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Tue, 13 Oct 2020 19:28:34 +0800 Subject: [PATCH 109/124] agent: Use `?` instead of `match` when the error returns directly It's more clear and more readable. Signed-off-by: Tim Zhang --- src/agent/src/sandbox.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index ac80f5be0f..ee5ab08f98 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -143,16 +143,10 @@ impl Sandbox { // It's assumed that caller is calling this method after // acquiring a lock on sandbox. pub fn unset_and_remove_sandbox_storage(&mut self, path: &str) -> Result<()> { - match self.unset_sandbox_storage(path) { - Ok(res) => { - if res { - return self.remove_sandbox_storage(path); - } - } - Err(err) => { - return Err(err); - } + if self.unset_sandbox_storage(path)? { + return self.remove_sandbox_storage(path); } + Ok(()) } From 1fb67309843103952fcb759089614aa59233d954 Mon Sep 17 00:00:00 2001 From: Tim Zhang Date: Wed, 14 Oct 2020 10:18:00 +0800 Subject: [PATCH 110/124] agent: remove `unwrap()` for `e.as_errno()` Use `{:?}` to print `e.as_errno()` instead of using `{}` to print `e.as_errno().unwrap().desc()`. Avoid panic only caused by error's content. Signed-off-by: Tim Zhang --- src/agent/rustjail/src/container.rs | 4 ++-- src/agent/rustjail/src/mount.rs | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index 9add8d152e..7c166fd24c 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -397,7 +397,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> { ns.r#type.clone(), ns.path.clone() ); - log_child!(cfd_log, "error is : {}", e.as_errno().unwrap().desc()); + log_child!(cfd_log, "error is : {:?}", e.as_errno()); e })?; @@ -1080,7 +1080,7 @@ fn get_pid_namespace(logger: &Logger, linux: &Linux) -> Result> { ns.r#type.clone(), ns.path.clone() ); - error!(logger, "error is : {}", e.as_errno().unwrap().desc()); + error!(logger, "error is : {:?}", e.as_errno()); e })?; diff --git a/src/agent/rustjail/src/mount.rs b/src/agent/rustjail/src/mount.rs index cc3b928a4a..1942fcc5b8 100644 --- a/src/agent/rustjail/src/mount.rs +++ b/src/agent/rustjail/src/mount.rs @@ -712,9 +712,9 @@ fn mount_from( let _ = stat::stat(dest.as_str()).map_err(|e| { log_child!( cfd_log, - "dest stat error. {}: {}", + "dest stat error. {}: {:?}", dest.as_str(), - e.as_errno().unwrap().desc() + e.as_errno() ) }); @@ -726,7 +726,7 @@ fn mount_from( Some(d.as_str()), ) .map_err(|e| { - log_child!(cfd_log, "mount error: {}", e.as_errno().unwrap().desc()); + log_child!(cfd_log, "mount error: {:?}", e.as_errno()); e })?; @@ -748,12 +748,7 @@ fn mount_from( None::<&str>, ) .map_err(|e| { - log_child!( - cfd_log, - "remout {}: {}", - dest.as_str(), - e.as_errno().unwrap().desc() - ); + log_child!(cfd_log, "remout {}: {:?}", dest.as_str(), e.as_errno()); e })?; } From ef11213a4e0fcf9f89779dbeb9a4e770369918d8 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 14 Oct 2020 20:46:00 -0700 Subject: [PATCH 111/124] config: make virtio-fs part of standard kernel Basic virtio-fs support has made it upstream in the Linux kernel, as well as in QEMU and Cloud Hypervisor. Let's go ahead and add it to the standard configuration. Since the device driver / DAX handling is still in progress for upstream, we will want to still build a seperate experimental kernel for those who are comfortable trading off bleeding edge stability/kernel updates for improved FIO numbers. Fixes: #963 Signed-off-by: Eric Ernst --- .../configs/fragments/common/experimental/virtio-fs.conf | 3 --- tools/packaging/kernel/configs/fragments/common/fs.conf | 4 ++++ tools/packaging/kernel/kata_config_version | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 tools/packaging/kernel/configs/fragments/common/experimental/virtio-fs.conf diff --git a/tools/packaging/kernel/configs/fragments/common/experimental/virtio-fs.conf b/tools/packaging/kernel/configs/fragments/common/experimental/virtio-fs.conf deleted file mode 100644 index ed1810f135..0000000000 --- a/tools/packaging/kernel/configs/fragments/common/experimental/virtio-fs.conf +++ /dev/null @@ -1,3 +0,0 @@ -# virtio-fs support -CONFIG_VIRTIO_FS=y -CONFIG_FUSE_FS=y diff --git a/tools/packaging/kernel/configs/fragments/common/fs.conf b/tools/packaging/kernel/configs/fragments/common/fs.conf index 28d9019ee2..898091db9e 100644 --- a/tools/packaging/kernel/configs/fragments/common/fs.conf +++ b/tools/packaging/kernel/configs/fragments/common/fs.conf @@ -49,3 +49,7 @@ CONFIG_AIO=y CONFIG_OVERLAY_FS=y CONFIG_OVERLAY_FS_INDEX=y CONFIG_OVERLAY_FS_REDIRECT_DIR=y + +# virtio-fs driver support: +CONFIG_VIRTIO_FS=y +CONFIG_FUSE_FS=y diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 76a8b2b703..871727de1f 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -83 +84 From ee59378232bf6e6168a642d2f8e3ec909eabbe13 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 15 Oct 2020 15:27:19 -0700 Subject: [PATCH 112/124] kernel: update to 5.4.71 vsock fix was backported to 5.4 stable, so we can drop this patch. Fixes: #973 Signed-off-by: Eric Ernst --- ...-Fix-race-condition-between-bind-and.patch | 49 ------------------- versions.yaml | 2 +- 2 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 tools/packaging/kernel/patches/5.4.x/0002-net-virtio_vsock-Fix-race-condition-between-bind-and.patch diff --git a/tools/packaging/kernel/patches/5.4.x/0002-net-virtio_vsock-Fix-race-condition-between-bind-and.patch b/tools/packaging/kernel/patches/5.4.x/0002-net-virtio_vsock-Fix-race-condition-between-bind-and.patch deleted file mode 100644 index a938d6552a..0000000000 --- a/tools/packaging/kernel/patches/5.4.x/0002-net-virtio_vsock-Fix-race-condition-between-bind-and.patch +++ /dev/null @@ -1,49 +0,0 @@ -From ac1956caf20f8ac0589f69b2d5fcc81e6ba7c71a Mon Sep 17 00:00:00 2001 -From: Sebastien Boeuf -Date: Thu, 13 Feb 2020 08:50:38 +0100 -Subject: [PATCH] net: virtio_vsock: Fix race condition between bind and listen - -Whenever the vsock backend on the host sends a packet through the RX -queue, it expects an answer on the TX queue. Unfortunately, there is one -case where the host side will hang waiting for the answer and will -effectively never recover. - -This issue happens when the guest side starts binding to the socket, -which insert a new bound socket into the list of already bound sockets. -At this time, we expect the guest to also start listening, which will -trigger the sk_state to move from TCP_CLOSE to TCP_LISTEN. The problem -occurs if the host side queued a RX packet and triggered an interrupt -right between the end of the binding process and the beginning of the -listening process. In this specific case, the function processing the -packet virtio_transport_recv_pkt() will find a bound socket, which means -it will hit the switch statement checking for the sk_state, but the -state won't be changed into TCP_LISTEN yet, which leads the code to pick -the default statement. This default statement will only free the buffer, -while it should also respond to the host side, by sending a packet on -its TX queue. - -In order to simply fix this unfortunate chain of events, it is important -that in case the default statement is entered, and because at this stage -we know the host side is waiting for an answer, we must send back a -packet containing the operation VIRTIO_VSOCK_OP_RST. - -Signed-off-by: Sebastien Boeuf ---- - net/vmw_vsock/virtio_transport_common.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c -index fb2060dffb0a..696e9a03ad0f 100644 ---- a/net/vmw_vsock/virtio_transport_common.c -+++ b/net/vmw_vsock/virtio_transport_common.c -@@ -1127,6 +1127,7 @@ void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt) - virtio_transport_free_pkt(pkt); - break; - default: -+ (void)virtio_transport_reset_no_sock(pkt); - virtio_transport_free_pkt(pkt); - break; - } --- -2.20.1 - diff --git a/versions.yaml b/versions.yaml index eb5982ca87..f9ba333f0a 100644 --- a/versions.yaml +++ b/versions.yaml @@ -155,7 +155,7 @@ assets: url: "https://cdn.kernel.org/pub/linux/kernel/v4.x/" uscan-url: >- https://mirrors.edge.kernel.org/pub/linux/kernel/v5.x/linux-(5\.4\..+)\.tar\.gz - version: "v5.4.60" + version: "v5.4.71" kernel-experimental: description: "Linux kernel with virtio-fs support" From 6511ffe89d7838f34a06ca409e059f021ba12328 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 16:54:34 +0100 Subject: [PATCH 113/124] tools: Fix comment in agent-ctl Correct a comment in the agent control tool. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index 47e6a85bd7..3950e7005c 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -1179,7 +1179,7 @@ fn agent_cmd_sandbox_list_routes( #[inline] fn builtin_cmd_repeat(_cfg: &Config, _options: &mut Options, _args: &str) -> (Result<()>, bool) { - // XXX: NOP implementation. Due to the way repeat has to work, providing + // XXX: NOP implementation. Due to the way repeat has to work, providing a // handler like this is "too late" to be useful. However, a handler // is required as "repeat" is a valid command. // From 0c432153df08d1751c98cb8e2245f0b8760d7d6d Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 17:05:19 +0100 Subject: [PATCH 114/124] tools: Rename agent-ctl command to GetGuestDetails Rename the `GuestDetails` command to `GetGuestDetails` to match the actual agent API name. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/client.rs | 6 +++--- tools/agent-ctl/src/main.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index 3950e7005c..92d5b042df 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -106,9 +106,9 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ fp: agent_cmd_container_exec, }, AgentCmd { - name: "GuestDetails", + name: "GetGuestDetails", st: ServiceType::Agent, - fp: agent_cmd_sandbox_guest_details, + fp: agent_cmd_sandbox_get_guest_details, }, AgentCmd { name: "ListInterfaces", @@ -940,7 +940,7 @@ fn agent_cmd_container_start( Ok(()) } -fn agent_cmd_sandbox_guest_details( +fn agent_cmd_sandbox_get_guest_details( cfg: &Config, client: &AgentServiceClient, _health: &HealthClient, diff --git a/tools/agent-ctl/src/main.rs b/tools/agent-ctl/src/main.rs index c9514772a4..e36ee22783 100644 --- a/tools/agent-ctl/src/main.rs +++ b/tools/agent-ctl/src/main.rs @@ -65,7 +65,7 @@ fn make_examples_text(program_name: &str) -> String { - Query the agent environment: - $ {program} connect --server-address "{vsock_server_address}" --cmd GuestDetails + $ {program} connect --server-address "{vsock_server_address}" --cmd GetGuestDetails - List all available (built-in and Kata Agent API) commands: @@ -85,7 +85,7 @@ fn make_examples_text(program_name: &str) -> String { - Query guest details forever: - $ {program} connect --server-address "{vsock_server_address}" --repeat -1 --cmd GuestDetails + $ {program} connect --server-address "{vsock_server_address}" --repeat -1 --cmd GetGuestDetails - Send a 'SIGUSR1' signal to a container process: From 81fb2c9980c559727a8fbbc4e1316ec3aa5bdac0 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 17:13:08 +0100 Subject: [PATCH 115/124] tools: Log request in agent-ctl tool if debug enabled Display the API request before making the call so users can see what is sent to the agent. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/client.rs | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index 92d5b042df..0ec23d6abb 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -684,6 +684,8 @@ fn agent_cmd_health_check( // value unused req.set_service("".to_string()); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = health .check(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -707,6 +709,8 @@ fn agent_cmd_health_version( // value unused req.set_service("".to_string()); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = health .version(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -729,6 +733,8 @@ fn agent_cmd_sandbox_create( let sid = utils::get_option("sid", options, args); req.set_sandbox_id(sid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .create_sandbox(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -748,6 +754,8 @@ fn agent_cmd_sandbox_destroy( ) -> Result<()> { let req = DestroySandboxRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .destroy_sandbox(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -778,6 +786,8 @@ fn agent_cmd_container_create( req.set_exec_id(exec_id); req.set_OCI(grpc_spec); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .create_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -801,6 +811,8 @@ fn agent_cmd_container_remove( req.set_container_id(cid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .remove_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -838,6 +850,8 @@ fn agent_cmd_container_exec( req.set_exec_id(exec_id); req.set_process(process); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .exec_process(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -861,6 +875,8 @@ fn agent_cmd_container_stats( req.set_container_id(cid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .stats_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -884,6 +900,8 @@ fn agent_cmd_container_pause( req.set_container_id(cid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .pause_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -907,6 +925,8 @@ fn agent_cmd_container_resume( req.set_container_id(cid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .resume_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -930,6 +950,8 @@ fn agent_cmd_container_start( req.set_container_id(cid); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .start_container(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -951,6 +973,8 @@ fn agent_cmd_sandbox_get_guest_details( req.set_mem_block_size(true); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .get_guest_details(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -981,6 +1005,8 @@ fn agent_cmd_container_list_processes( req.set_container_id(cid); req.set_format(list_format); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .list_processes(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1006,6 +1032,8 @@ fn agent_cmd_container_wait_process( req.set_container_id(cid); req.set_exec_id(exec_id); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .wait_process(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1041,6 +1069,8 @@ fn agent_cmd_container_signal_process( req.set_exec_id(exec_id); req.set_signal(signum as u32); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .signal_process(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1060,6 +1090,8 @@ fn agent_cmd_sandbox_tracing_start( ) -> Result<()> { let req = StartTracingRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .start_tracing(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1079,6 +1111,8 @@ fn agent_cmd_sandbox_tracing_stop( ) -> Result<()> { let req = StopTracingRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .stop_tracing(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1098,6 +1132,7 @@ fn agent_cmd_sandbox_update_interface( ) -> Result<()> { let req = UpdateInterfaceRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); let reply = client .update_interface(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1123,6 +1158,8 @@ fn agent_cmd_sandbox_update_routes( ) -> Result<()> { let req = UpdateRoutesRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .update_routes(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1148,6 +1185,8 @@ fn agent_cmd_sandbox_list_interfaces( ) -> Result<()> { let req = ListInterfacesRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .list_interfaces(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; @@ -1167,6 +1206,8 @@ fn agent_cmd_sandbox_list_routes( ) -> Result<()> { let req = ListRoutesRequest::default(); + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + let reply = client .list_routes(&req, cfg.timeout_nano) .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; From 46cfed502576345725ae98a7ad47572f62319727 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 17:16:30 +0100 Subject: [PATCH 116/124] tools: Remove commented out code in agent-ctl Remove a few lines of commented out code. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/src/client.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index 0ec23d6abb..e2ecb9cead 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -1140,9 +1140,6 @@ fn agent_cmd_sandbox_update_interface( // FIXME: Implement 'UpdateInterface' fully. eprintln!("FIXME: 'UpdateInterface' not fully implemented"); - // let if = ...; - // req.set_interface(if); - info!(sl!(), "response received"; "response" => format!("{:?}", reply)); @@ -1167,9 +1164,6 @@ fn agent_cmd_sandbox_update_routes( // FIXME: Implement 'UpdateRoutes' fully. eprintln!("FIXME: 'UpdateRoutes' not fully implemented"); - // let routes = ...; - // req.set_routes(routes); - info!(sl!(), "response received"; "response" => format!("{:?}", reply)); From e6f7ddd9a2ba6eaeadf6b7a8fd09777bb46c1982 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 17:30:22 +0100 Subject: [PATCH 117/124] tools: Make agent-ctl support more APIs Added new `agent-ctl` commands to allow the following agent API calls to be made: - `AddARPNeighborsRequest` - `CloseStdinRequest` - `CopyFileRequest` - `GetMetricsRequest` - `GetOOMEventRequest` - `MemHotplugByProbeRequest` - `OnlineCPUMemRequest` - `ReadStreamRequest` - `ReseedRandomDevRequest` - `SetGuestDateTimeRequest` - `TtyWinResizeRequest` - `UpdateContainerRequest` - `WriteStreamRequest` Fixes: #969. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/Cargo.lock | 10 +- tools/agent-ctl/Cargo.toml | 2 + tools/agent-ctl/src/client.rs | 593 ++++++++++++++++++++++++++++++++++ tools/agent-ctl/src/utils.rs | 14 + 4 files changed, 618 insertions(+), 1 deletion(-) diff --git a/tools/agent-ctl/Cargo.lock b/tools/agent-ctl/Cargo.lock index 897425ab5c..1d37399fa0 100644 --- a/tools/agent-ctl/Cargo.lock +++ b/tools/agent-ctl/Cargo.lock @@ -257,6 +257,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "humantime" version = "2.0.1" @@ -274,7 +280,9 @@ name = "kata-agent-ctl" version = "0.0.1" dependencies = [ "anyhow", + "byteorder", "clap", + "hex", "humantime", "lazy_static", "libc", @@ -435,7 +443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52" dependencies = [ "libc", - "nix 0.17.0", + "nix 0.18.0", ] [[package]] diff --git a/tools/agent-ctl/Cargo.toml b/tools/agent-ctl/Cargo.toml index fcaa98a71f..ae8a752dbc 100644 --- a/tools/agent-ctl/Cargo.toml +++ b/tools/agent-ctl/Cargo.toml @@ -17,6 +17,8 @@ oci = { path = "../../src/agent/oci" } clap = "2.33.0" lazy_static = "1.4.0" anyhow = "1.0.31" +hex = "0.4.2" +byteorder = "1.3.4" logging = { path = "../../pkg/logging" } slog = "2.5.2" diff --git a/tools/agent-ctl/src/client.rs b/tools/agent-ctl/src/client.rs index e2ecb9cead..ac38bcb381 100644 --- a/tools/agent-ctl/src/client.rs +++ b/tools/agent-ctl/src/client.rs @@ -8,6 +8,7 @@ use crate::types::{Config, Options}; use crate::utils; use anyhow::{anyhow, Result}; +use byteorder::ByteOrder; use nix::sys::socket::{connect, socket, AddressFamily, SockAddr, SockFlag, SockType, UnixAddr}; use protocols::agent::*; use protocols::agent_ttrpc::*; @@ -75,6 +76,11 @@ const DEFAULT_PS_FORMAT: &str = "json"; const ERR_API_FAILED: &str = "API failed"; static AGENT_CMDS: &'static [AgentCmd] = &[ + AgentCmd { + name: "AddARPNeighbors", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_add_arp_neighbors, + }, AgentCmd { name: "Check", st: ServiceType::Health, @@ -85,6 +91,16 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Health, fp: agent_cmd_health_version, }, + AgentCmd { + name: "CloseStdin", + st: ServiceType::Agent, + fp: agent_cmd_container_close_stdin, + }, + AgentCmd { + name: "CopyFile", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_copy_file, + }, AgentCmd { name: "CreateContainer", st: ServiceType::Agent, @@ -110,6 +126,16 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_sandbox_get_guest_details, }, + AgentCmd { + name: "GetMetrics", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_get_metrics, + }, + AgentCmd { + name: "GetOOMEvent", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_get_oom_event, + }, AgentCmd { name: "ListInterfaces", st: ServiceType::Agent, @@ -125,11 +151,36 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_container_list_processes, }, + AgentCmd { + name: "MemHotplugByProbe", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_mem_hotplug_by_probe, + }, + AgentCmd { + name: "OnlineCPUMem", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_online_cpu_mem, + }, AgentCmd { name: "PauseContainer", st: ServiceType::Agent, fp: agent_cmd_container_pause, }, + AgentCmd { + name: "ReadStderr", + st: ServiceType::Agent, + fp: agent_cmd_container_read_stderr, + }, + AgentCmd { + name: "ReadStdout", + st: ServiceType::Agent, + fp: agent_cmd_container_read_stdout, + }, + AgentCmd { + name: "ReseedRandomDev", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_reseed_random_dev, + }, AgentCmd { name: "RemoveContainer", st: ServiceType::Agent, @@ -140,6 +191,11 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_container_resume, }, + AgentCmd { + name: "SetGuestDateTime", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_set_guest_date_time, + }, AgentCmd { name: "SignalProcess", st: ServiceType::Agent, @@ -165,6 +221,16 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_sandbox_tracing_stop, }, + AgentCmd { + name: "TtyWinResize", + st: ServiceType::Agent, + fp: agent_cmd_container_tty_win_resize, + }, + AgentCmd { + name: "UpdateContainer", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_update_container, + }, AgentCmd { name: "UpdateInterface", st: ServiceType::Agent, @@ -180,6 +246,11 @@ static AGENT_CMDS: &'static [AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_container_wait_process, }, + AgentCmd { + name: "WriteStdin", + st: ServiceType::Agent, + fp: agent_cmd_container_write_stdin, + }, ]; static BUILTIN_CMDS: &'static [BuiltinCmd] = &[ @@ -1212,6 +1283,528 @@ fn agent_cmd_sandbox_list_routes( Ok(()) } +fn agent_cmd_container_tty_win_resize( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = TtyWinResizeRequest::default(); + + let cid = utils::get_option("cid", options, args); + let exec_id = utils::get_option("exec_id", options, args); + + req.set_container_id(cid); + req.set_exec_id(exec_id); + + let rows_str = utils::get_option("row", options, args); + + if rows_str != "" { + let rows = rows_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid row size"))?; + req.set_row(rows); + } + + let cols_str = utils::get_option("column", options, args); + + if cols_str != "" { + let cols = cols_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid column size"))?; + + req.set_column(cols); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .tty_win_resize(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_container_close_stdin( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = CloseStdinRequest::default(); + + let cid = utils::get_option("cid", options, args); + let exec_id = utils::get_option("exec_id", options, args); + + req.set_container_id(cid); + req.set_exec_id(exec_id); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .close_stdin(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_container_read_stdout( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = ReadStreamRequest::default(); + + let cid = utils::get_option("cid", options, args); + let exec_id = utils::get_option("exec_id", options, args); + + req.set_container_id(cid); + req.set_exec_id(exec_id); + + let length_str = utils::get_option("len", options, args); + + if length_str != "" { + let length = length_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid length"))?; + req.set_len(length); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .read_stdout(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_container_read_stderr( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = ReadStreamRequest::default(); + + let cid = utils::get_option("cid", options, args); + let exec_id = utils::get_option("exec_id", options, args); + + req.set_container_id(cid); + req.set_exec_id(exec_id); + + let length_str = utils::get_option("len", options, args); + + if length_str != "" { + let length = length_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid length"))?; + req.set_len(length); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .read_stderr(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_container_write_stdin( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = WriteStreamRequest::default(); + + let cid = utils::get_option("cid", options, args); + let exec_id = utils::get_option("exec_id", options, args); + + let str_data = utils::get_option("data", options, args); + let data = utils::str_to_bytes(&str_data)?; + + req.set_container_id(cid); + req.set_exec_id(exec_id); + req.set_data(data.to_vec()); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .write_stdin(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_get_metrics( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + _options: &mut Options, + _args: &str, +) -> Result<()> { + let req = GetMetricsRequest::default(); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .get_metrics(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_get_oom_event( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + _options: &mut Options, + _args: &str, +) -> Result<()> { + let req = GetOOMEventRequest::default(); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .get_oom_event(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_copy_file( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = CopyFileRequest::default(); + + let path = utils::get_option("path", options, args); + if path != "" { + req.set_path(path); + } + + let file_size_str = utils::get_option("file_size", options, args); + + if file_size_str != "" { + let file_size = file_size_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid file_size"))?; + + req.set_file_size(file_size); + } + + let file_mode_str = utils::get_option("file_mode", options, args); + + if file_mode_str != "" { + let file_mode = file_mode_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid file_mode"))?; + + req.set_file_mode(file_mode); + } + + let dir_mode_str = utils::get_option("dir_mode", options, args); + + if dir_mode_str != "" { + let dir_mode = dir_mode_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid dir_mode"))?; + + req.set_dir_mode(dir_mode); + } + + let uid_str = utils::get_option("uid", options, args); + + if uid_str != "" { + let uid = uid_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid uid"))?; + + req.set_uid(uid); + } + + let gid_str = utils::get_option("gid", options, args); + + if gid_str != "" { + let gid = gid_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid gid"))?; + req.set_gid(gid); + } + + let offset_str = utils::get_option("offset", options, args); + + if offset_str != "" { + let offset = offset_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid offset"))?; + req.set_offset(offset); + } + + let data_str = utils::get_option("data", options, args); + if data_str != "" { + let data = utils::str_to_bytes(&data_str)?; + req.set_data(data.to_vec()); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .copy_file(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_reseed_random_dev( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = ReseedRandomDevRequest::default(); + + let str_data = utils::get_option("data", options, args); + let data = utils::str_to_bytes(&str_data)?; + + req.set_data(data.to_vec()); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .reseed_random_dev(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_online_cpu_mem( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = OnlineCPUMemRequest::default(); + + let wait_str = utils::get_option("wait", options, args); + + if wait_str != "" { + let wait = wait_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid wait bool"))?; + + req.set_wait(wait); + } + + let nb_cpus_str = utils::get_option("nb_cpus", options, args); + + if nb_cpus_str != "" { + let nb_cpus = nb_cpus_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid nb_cpus value"))?; + + req.set_nb_cpus(nb_cpus); + } + + let cpu_only_str = utils::get_option("cpu_only", options, args); + + if cpu_only_str != "" { + let cpu_only = cpu_only_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid cpu_only bool"))?; + + req.set_cpu_only(cpu_only); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .online_cpu_mem(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_set_guest_date_time( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = SetGuestDateTimeRequest::default(); + + let secs_str = utils::get_option("sec", options, args); + + if secs_str != "" { + let secs = secs_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid seconds"))?; + + req.set_Sec(secs); + } + + let usecs_str = utils::get_option("usec", options, args); + + if usecs_str != "" { + let usecs = usecs_str + .parse::() + .map_err(|e| anyhow!(e).context("invalid useconds"))?; + + req.set_Usec(usecs); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .set_guest_date_time(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_add_arp_neighbors( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + _options: &mut Options, + _args: &str, +) -> Result<()> { + let req = AddARPNeighborsRequest::default(); + + // FIXME: Implement fully. + eprintln!("FIXME: 'AddARPNeighbors' not fully implemented"); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .add_arp_neighbors(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_update_container( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = UpdateContainerRequest::default(); + + let cid = utils::get_option("cid", options, args); + + req.set_container_id(cid); + + // FIXME: Implement fully + eprintln!("FIXME: 'UpdateContainer' not fully implemented"); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .update_container(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + +fn agent_cmd_sandbox_mem_hotplug_by_probe( + cfg: &Config, + client: &AgentServiceClient, + _health: &HealthClient, + options: &mut Options, + args: &str, +) -> Result<()> { + let mut req = MemHotplugByProbeRequest::default(); + + // Expected to be a comma separated list of hex addresses + let addr_list = utils::get_option("memHotplugProbeAddr", options, args); + + if addr_list != "" { + let addrs: Vec = addr_list + // Convert into a list of string values. + .split(",") + // Convert each string element into a u8 array of bytes, ignoring + // those elements that fail the conversion. + .filter_map(|s| hex::decode(s.trim_start_matches("0x")).ok()) + // "Stretch" the u8 byte slice into one of length 8 + // (to allow each 8 byte chunk to be converted into a u64). + .map(|mut v| -> Vec { + v.resize(8, 0x0); + v + }) + // Convert the slice of u8 bytes into a u64 + .map(|b| byteorder::LittleEndian::read_u64(&b)) + .collect(); + + req.set_memHotplugProbeAddr(addrs); + } + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .mem_hotplug_by_probe(&req, cfg.timeout_nano) + .map_err(|e| anyhow!("{:?}", e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + #[inline] fn builtin_cmd_repeat(_cfg: &Config, _options: &mut Options, _args: &str) -> (Result<()>, bool) { // XXX: NOP implementation. Due to the way repeat has to work, providing a diff --git a/tools/agent-ctl/src/utils.rs b/tools/agent-ctl/src/utils.rs index 7dff5d8cb6..1a71fd6da3 100644 --- a/tools/agent-ctl/src/utils.rs +++ b/tools/agent-ctl/src/utils.rs @@ -408,3 +408,17 @@ pub fn get_grpc_spec(options: &mut Options, cid: &str) -> Result { Ok(oci_to_grpc(&bundle_dir, cid, &oci_spec)?) } + +pub fn str_to_bytes(s: &str) -> Result> { + let prefix = "hex:"; + + if s.starts_with(prefix) { + let hex_str = s.trim_start_matches(prefix); + + let decoded = hex::decode(hex_str).map_err(|e| anyhow!(e))?; + + Ok(decoded) + } else { + Ok(s.as_bytes().to_vec()) + } +} From c7bb1e2790bda4e3e918a7ac8e8e9923580947c2 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 15 Oct 2020 17:17:23 +0100 Subject: [PATCH 118/124] tools: Improve agent-ctl README Add a summary to help understand how to use the `agent-ctl` tool. Signed-off-by: James O. D. Hunt --- tools/agent-ctl/README.md | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tools/agent-ctl/README.md b/tools/agent-ctl/README.md index 862a0e3351..47a1776ee2 100644 --- a/tools/agent-ctl/README.md +++ b/tools/agent-ctl/README.md @@ -3,6 +3,11 @@ * [Overview](#overview) * [Audience and environment](#audience-and-environment) * [Full details](#full-details) +* [Code summary](#code-summary) +* [Running the tool](#running-the-tool) + * [Prerequisites](#prerequisites) + * [Connect to a real Kata Container](#connect-to-a-real-kata-container) + * [Run the tool and the agent in the same environment](#run-the-tool-and-the-agent-in-the-same-environment) ## Overview @@ -37,3 +42,80 @@ To see some examples, run: ```sh $ cargo run -- examples ``` + +## Code summary + +The table below summarises where to look to learn more about both this tool, +the agent protocol and the client and server implementations. + +| Description | File | Example RPC or function | Example summary | +|-|-|-|-| +| Protocol buffers definition of the Kata Containers Agent API protocol | [`agent.proto`](../../src/agent/protocols/protos/agent.proto) | `CreateContainer` | API to create a Kata container. | +| Agent Control (client) API calls | [`src/client.rs`](src/client.rs) | `agent_cmd_container_create()` | Agent Control tool function that calls the `CreateContainer` API. | +| Agent (server) API implementations | [`rpc.rs`](../../src/agent/src/rpc.rs) | `create_container()` | Server function that implements the `CreateContainers` API. | + +## Running the tool + +### Prerequisites + +It is necessary to create an OCI bundle to use the tool. The simplest method +is: + +```sh +$ bundle_dir="bundle" +$ rootfs_dir="$bundle_dir/rootfs" +$ image="busybox" +$ mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec) +$ sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf - +``` + +### Connect to a real Kata Container + +1. Start a Kata Container + +1. Establish the VSOCK guest CID number for the virtual machine: + + Assuming you are running a single QEMU based Kata Container, you can look + at the program arguments to find the (randomly-generated) `guest-cid=` option + value: + + ```sh + $ guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[^,][^,]*" | cut -d= -f2) + ``` + +1. Run the tool to connect to the agent: + + ```sh + $ cargo run -- -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails + ``` + + This examples makes two API calls: + + - It runs `Check` to see if the agent's RPC server is serving. + - It then runs `GetGuestDetails` to establish some details of the + environment the agent is running in. + +### Run the tool and the agent in the same environment + +> **Warnings:** +> +> - This method is **only** for testing and development! +> - Only continue if you are using a non-critical system +> (such as a freshly installed VM environment). + +1. Start the agent, specifying a local socket for it to communicate on: + + ```sh + $ sudo KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket target/x86_64-unknown-linux-musl/release/kata-agent + ``` + +1. Run the tool in the same environment: + + ```sh + $ cargo run -- -l debug connect --server-address "unix://@/tmp/foo.socket" --bundle-dir "$bundle_dir" -c Check -c GetGuestDetails + ``` + + > **Note:** + > + > The `@` in the server address is required - it denotes an abstract + > socket which the agent requires (see `unix(7)`). From 7347d43cf9b32aa70765987f11991ffd0bbe70f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 13 Oct 2020 18:14:41 +0200 Subject: [PATCH 119/124] packaging: Apply virtiofs performance related fixes to 5.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vivek Goyal found out that using "shared" thread pool, instead of "exclusive" results in better performance. Knowning that and with the plan to have virtio-fs as the default fs for the 2.0, let's bring this patch in for both 5.0 and 5.1. Fixes: #944 Signed-off-by: Fabiano Fidêncio --- ...rtiofsd-Used-glib-shared-thread-pool.patch | 60 +++++++++++++++++++ ...rtiofsd-Used-glib-shared-thread-pool.patch | 60 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 tools/packaging/qemu/patches/5.0.x/0003-virtiofsd-Used-glib-shared-thread-pool.patch create mode 100644 tools/packaging/qemu/patches/5.1.x/0006-virtiofsd-Used-glib-shared-thread-pool.patch diff --git a/tools/packaging/qemu/patches/5.0.x/0003-virtiofsd-Used-glib-shared-thread-pool.patch b/tools/packaging/qemu/patches/5.0.x/0003-virtiofsd-Used-glib-shared-thread-pool.patch new file mode 100644 index 0000000000..2e8113b8e7 --- /dev/null +++ b/tools/packaging/qemu/patches/5.0.x/0003-virtiofsd-Used-glib-shared-thread-pool.patch @@ -0,0 +1,60 @@ +From 04d325e86f79bd61f8fd50d45ff795aca0dd3404 Mon Sep 17 00:00:00 2001 +From: Vivek Goyal +Date: Mon, 21 Sep 2020 17:32:16 -0400 +Subject: [PATCH] virtiofsd: Used glib "shared" thread pool + +glib offers thread pools and it seems to support "exclusive" and "shared" +thread pools. + +https://developer.gnome.org/glib/stable/glib-Thread-Pools.html#g-thread-pool-new + +Currently we use "exlusive" thread pools but its performance seems to be +poor. I tried using "shared" thread pools and performance seems much +better. I posted performance results here. + +https://www.redhat.com/archives/virtio-fs/2020-September/msg00080.html + +So lets switch to shared thread pools. We can think of making it optional +once somebody can show in what cases exclusive thread pools offer better +results. For now, my simple performance tests across the board see +better results with shared thread pools. + +Signed-off-by: Vivek Goyal +Message-Id: <20200921213216.GE13362@redhat.com> +Reviewed-by: Stefan Hajnoczi +Signed-off-by: Dr. David Alan Gilbert + With seccomp fix from Miklos +--- + tools/virtiofsd/fuse_virtio.c | 2 +- + tools/virtiofsd/seccomp.c | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/tools/virtiofsd/fuse_virtio.c b/tools/virtiofsd/fuse_virtio.c +index 9e5537506c..d5c8e98253 100644 +--- a/tools/virtiofsd/fuse_virtio.c ++++ b/tools/virtiofsd/fuse_virtio.c +@@ -588,7 +588,7 @@ static void *fv_queue_thread(void *opaque) + struct fuse_session *se = qi->virtio_dev->se; + GThreadPool *pool; + +- pool = g_thread_pool_new(fv_queue_worker, qi, se->thread_pool_size, TRUE, ++ pool = g_thread_pool_new(fv_queue_worker, qi, se->thread_pool_size, FALSE, + NULL); + if (!pool) { + fuse_log(FUSE_LOG_ERR, "%s: g_thread_pool_new failed\n", __func__); +diff --git a/tools/virtiofsd/seccomp.c b/tools/virtiofsd/seccomp.c +index 19fee60011..eb9af8265f 100644 +--- a/tools/virtiofsd/seccomp.c ++++ b/tools/virtiofsd/seccomp.c +@@ -93,6 +93,8 @@ static const int syscall_whitelist[] = { + SCMP_SYS(rt_sigaction), + SCMP_SYS(rt_sigprocmask), + SCMP_SYS(rt_sigreturn), ++ SCMP_SYS(sched_getattr), ++ SCMP_SYS(sched_setattr), + SCMP_SYS(sendmsg), + SCMP_SYS(setresgid), + SCMP_SYS(setresuid), +-- +2.28.0 + diff --git a/tools/packaging/qemu/patches/5.1.x/0006-virtiofsd-Used-glib-shared-thread-pool.patch b/tools/packaging/qemu/patches/5.1.x/0006-virtiofsd-Used-glib-shared-thread-pool.patch new file mode 100644 index 0000000000..2e8113b8e7 --- /dev/null +++ b/tools/packaging/qemu/patches/5.1.x/0006-virtiofsd-Used-glib-shared-thread-pool.patch @@ -0,0 +1,60 @@ +From 04d325e86f79bd61f8fd50d45ff795aca0dd3404 Mon Sep 17 00:00:00 2001 +From: Vivek Goyal +Date: Mon, 21 Sep 2020 17:32:16 -0400 +Subject: [PATCH] virtiofsd: Used glib "shared" thread pool + +glib offers thread pools and it seems to support "exclusive" and "shared" +thread pools. + +https://developer.gnome.org/glib/stable/glib-Thread-Pools.html#g-thread-pool-new + +Currently we use "exlusive" thread pools but its performance seems to be +poor. I tried using "shared" thread pools and performance seems much +better. I posted performance results here. + +https://www.redhat.com/archives/virtio-fs/2020-September/msg00080.html + +So lets switch to shared thread pools. We can think of making it optional +once somebody can show in what cases exclusive thread pools offer better +results. For now, my simple performance tests across the board see +better results with shared thread pools. + +Signed-off-by: Vivek Goyal +Message-Id: <20200921213216.GE13362@redhat.com> +Reviewed-by: Stefan Hajnoczi +Signed-off-by: Dr. David Alan Gilbert + With seccomp fix from Miklos +--- + tools/virtiofsd/fuse_virtio.c | 2 +- + tools/virtiofsd/seccomp.c | 2 ++ + 2 files changed, 3 insertions(+), 1 deletion(-) + +diff --git a/tools/virtiofsd/fuse_virtio.c b/tools/virtiofsd/fuse_virtio.c +index 9e5537506c..d5c8e98253 100644 +--- a/tools/virtiofsd/fuse_virtio.c ++++ b/tools/virtiofsd/fuse_virtio.c +@@ -588,7 +588,7 @@ static void *fv_queue_thread(void *opaque) + struct fuse_session *se = qi->virtio_dev->se; + GThreadPool *pool; + +- pool = g_thread_pool_new(fv_queue_worker, qi, se->thread_pool_size, TRUE, ++ pool = g_thread_pool_new(fv_queue_worker, qi, se->thread_pool_size, FALSE, + NULL); + if (!pool) { + fuse_log(FUSE_LOG_ERR, "%s: g_thread_pool_new failed\n", __func__); +diff --git a/tools/virtiofsd/seccomp.c b/tools/virtiofsd/seccomp.c +index 19fee60011..eb9af8265f 100644 +--- a/tools/virtiofsd/seccomp.c ++++ b/tools/virtiofsd/seccomp.c +@@ -93,6 +93,8 @@ static const int syscall_whitelist[] = { + SCMP_SYS(rt_sigaction), + SCMP_SYS(rt_sigprocmask), + SCMP_SYS(rt_sigreturn), ++ SCMP_SYS(sched_getattr), ++ SCMP_SYS(sched_setattr), + SCMP_SYS(sendmsg), + SCMP_SYS(setresgid), + SCMP_SYS(setresuid), +-- +2.28.0 + From da9bfb27ed6586baa45efc54e91740d9eaec06ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 13 Oct 2020 18:19:40 +0200 Subject: [PATCH 120/124] runtime: Pass `--thread-pool-size=1` to virtiofsd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dave Gilbert brough up that passing --thread-pool-size=1 to virtiofsd may result in a performance improvement especially when using `cache=none`. While our current default is `cache=auto`, Dave mentioned that he seems no harm in having it set and he also mentiond that it may use a lot less stack space on aarch/arm. Fixes: #943 Signed-off-by: Fabiano Fidêncio --- src/runtime/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index a5b8d4ef6f..685e7784ad 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -197,7 +197,7 @@ DEFVIRTIOFSCACHE ?= auto # # see `virtiofsd -h` for possible options. # Make sure you quote args. -DEFVIRTIOFSEXTRAARGS ?= [] +DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\"] DEFENABLEIOTHREADS := false DEFENABLEMEMPREALLOC := false DEFENABLEHUGEPAGES := false From 08361c5948f3e00550f464a9a3af888069b291b1 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Tue, 13 Oct 2020 14:30:35 +0800 Subject: [PATCH 121/124] runtime: enable virtiofs by default We've been shipping it for a long time. It's time to make it default replacing the old obsolet 9pfs. Fixes: #935 Signed-off-by: Peng Tao --- src/runtime/Makefile | 28 +- .../configuration-qemu-virtiofs.toml.in | 474 ------------------ .../cli/config/configuration-qemu.toml.in | 2 +- 3 files changed, 2 insertions(+), 502 deletions(-) delete mode 100644 src/runtime/cli/config/configuration-qemu-virtiofs.toml.in diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 685e7784ad..5a34b12139 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -99,7 +99,6 @@ GENERATED_VARS = \ VERSION \ CONFIG_ACRN_IN \ CONFIG_QEMU_IN \ - CONFIG_QEMU_VIRTIOFS_IN \ CONFIG_CLH_IN \ CONFIG_FC_IN \ $(USER_VARS) @@ -128,13 +127,12 @@ HYPERVISOR_FC = firecracker JAILER_FC = jailer HYPERVISOR_QEMU = qemu HYPERVISOR_CLH = cloud-hypervisor -HYPERVISOR_QEMU_VIRTIOFS = qemu-virtiofs # Determines which hypervisor is specified in $(CONFIG_FILE). DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU) # List of hypervisors this build system can generate configuration for. -HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_QEMU_VIRTIOFS) $(HYPERVISOR_CLH) +HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"] @@ -265,28 +263,6 @@ ifneq (,$(QEMUCMD)) KERNELPATH = $(KERNELDIR)/$(KERNELNAME) endif -ifneq (,$(QEMUVIRTIOFSCMD)) - KNOWN_HYPERVISORS += $(HYPERVISOR_QEMU_VIRTIOFS) - - CONFIG_FILE_QEMU_VIRTIOFS = configuration-qemu-virtiofs.toml - CONFIG_QEMU_VIRTIOFS = $(CLI_DIR)/config/$(CONFIG_FILE_QEMU_VIRTIOFS) - CONFIG_QEMU_VIRTIOFS_IN = $(CONFIG_QEMU_VIRTIOFS).in - - CONFIG_PATH_QEMU_VIRTIOFS = $(abspath $(CONFDIR)/$(CONFIG_FILE_QEMU_VIRTIOFS)) - CONFIG_PATHS += $(CONFIG_PATH_QEMU_VIRTIOFS) - - SYSCONFIG_QEMU_VIRTIOFS = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_QEMU_VIRTIOFS)) - SYSCONFIG_PATHS += $(SYSCONFIG_QEMU_VIRTIOFS) - - CONFIGS += $(CONFIG_QEMU_VIRTIOFS) - - # qemu-specific options (all should be suffixed by "_QEMU") - DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS := virtio-fs - DEFNETWORKMODEL_QEMU := tcfilter - KERNELNAMEVIRTIOFS = $(call MAKE_KERNEL_VIRTIOFS_NAME,$(KERNELTYPE)) - KERNELVIRTIOFSPATH = $(KERNELDIR)/$(KERNELNAMEVIRTIOFS) -endif - ifneq (,$(CLHCMD)) KNOWN_HYPERVISORS += $(HYPERVISOR_CLH) @@ -409,7 +385,6 @@ USER_VARS += CONFIG_CLH_IN USER_VARS += CONFIG_FC_IN USER_VARS += CONFIG_PATH USER_VARS += CONFIG_QEMU_IN -USER_VARS += CONFIG_QEMU_VIRTIOFS_IN USER_VARS += DESTDIR USER_VARS += DEFAULT_HYPERVISOR USER_VARS += DEFENABLEMSWAP @@ -490,7 +465,6 @@ USER_VARS += DEFDISABLEBLOCK USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN USER_VARS += DEFBLOCKSTORAGEDRIVER_FC USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU -USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU_VIRTIOFS USER_VARS += DEFSHAREDFS USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS USER_VARS += DEFVIRTIOFSDAEMON diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in deleted file mode 100644 index 41186907dd..0000000000 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ /dev/null @@ -1,474 +0,0 @@ -# Copyright (c) 2017-2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -# XXX: WARNING: this file is auto-generated. -# XXX: -# XXX: Source file: "@CONFIG_QEMU_VIRTIOFS_IN@" -# XXX: Project: -# XXX: Name: @PROJECT_NAME@ -# XXX: Type: @PROJECT_TYPE@ - -[hypervisor.qemu] -path = "@QEMUVIRTIOFSPATH@" -kernel = "@KERNELVIRTIOFSPATH@" -image = "@IMAGEPATH@" -machine_type = "@MACHINETYPE@" - -# List of valid annotation names for the hypervisor -# Each member of the list is a regular expression, which is the base name -# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -# The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFENABLEANNOTATIONS@ -enable_annotations = @DEFENABLEANNOTATIONS@ - -# List of valid annotations values for the hypervisor -# Each member of the list is a path pattern as described by glob(3). -# The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @QEMUVALIDHYPERVISORPATHS@ -valid_hypervisor_paths = @QEMUVALIDHYPERVISORPATHS@ - -# Optional space-separated list of options to pass to the guest kernel. -# For example, use `kernel_params = "vsyscall=emulate"` if you are having -# trouble running pre-2.15 glibc. -# -# WARNING: - any parameter specified here will take priority over the default -# parameter value of the same name used to start the virtual machine. -# Do not set values here unless you understand the impact of doing so as you -# may stop the virtual machine from booting. -# To see the list of default parameters, enable hypervisor debug, create a -# container and look for 'default-kernel-parameters' log entries. -kernel_params = "@KERNELPARAMS@" - -# Path to the firmware. -# If you want that qemu uses the default firmware leave this option empty -firmware = "@FIRMWAREPATH@" - -# Machine accelerators -# comma-separated list of machine accelerators to pass to the hypervisor. -# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` -machine_accelerators="@MACHINEACCELERATORS@" - -# CPU features -# comma-separated list of cpu features to pass to the cpu -# For example, `cpu_features = "pmu=off,vmx=off" -cpu_features="@CPUFEATURES@" - -# Default number of vCPUs per SB/VM: -# unspecified or 0 --> will be set to @DEFVCPUS@ -# < 0 --> will be set to the actual number of physical cores -# > 0 <= number of physical cores --> will be set to the specified number -# > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 - -# Default maximum number of vCPUs per SB/VM: -# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number -# of vCPUs supported by KVM if that number is exceeded -# > 0 <= number of physical cores --> will be set to the specified number -# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number -# of vCPUs supported by KVM if that number is exceeded -# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when -# the actual number of physical cores is greater than it. -# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU -# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs -# can be added to a SB/VM, but the memory footprint will be big. Another example, with -# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of -# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, -# unless you know what are you doing. -# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. -default_maxvcpus = @DEFMAXVCPUS@ - -# Bridges can be used to hot plug devices. -# Limitations: -# * Currently only pci bridges are supported -# * Until 30 devices per bridge can be hot plugged. -# * Until 5 PCI bridges can be cold plugged per VM. -# This limitation could be a bug in qemu or in the kernel -# Default number of bridges per SB/VM: -# unspecified or 0 --> will be set to @DEFBRIDGES@ -# > 1 <= 5 --> will be set to the specified number -# > 5 --> will be set to 5 -default_bridges = @DEFBRIDGES@ - -# Default memory size in MiB for SB/VM. -# If unspecified then it will be set @DEFMEMSZ@ MiB. -default_memory = @DEFMEMSZ@ -# -# Default memory slots per SB/VM. -# If unspecified then it will be set @DEFMEMSLOTS@. -# This is will determine the times that memory will be hotadded to sandbox/VM. -#memory_slots = @DEFMEMSLOTS@ - -# The size in MiB will be plused to max memory of hypervisor. -# It is the memory address space for the NVDIMM devie. -# If set block storage driver (block_device_driver) to "nvdimm", -# should set memory_offset to the size of block device. -# Default 0 -#memory_offset = 0 - -# Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's -# root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, -# 9pfs is used instead to pass the rootfs. -disable_block_device_use = @DEFDISABLEBLOCK@ - -# Shared file system type: -# - virtio-fs (default) -# - virtio-9p -shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" - -# Path to vhost-user-fs daemon. -virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" - -# List of valid annotations values for the virtiofs daemon -# The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVALIDVIRTIOFSDAEMONPATHS@ -valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ - -# Default size of DAX cache in MiB -virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ - -# Extra args for virtiofsd daemon -# -# Format example: -# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] -# -# see `virtiofsd -h` for possible options. -virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@ - -# Cache mode: -# -# - none -# Metadata, data, and pathname lookup are not cached in guest. They are -# always fetched from host and any changes are immediately pushed to host. -# -# - auto -# Metadata and pathname lookup cache expires after a configured amount of -# time (default is 1 second). Data is cached while the file is open (close -# to open consistency). -# -# - always -# Metadata, data, and pathname lookup are cached in guest and never expire. -virtio_fs_cache = "@DEFVIRTIOFSCACHE@" - -# Block storage driver to be used for the hypervisor in case the container -# rootfs is backed by a block device. This is virtio-scsi, virtio-blk -# or nvdimm. -block_device_driver = "@DEFBLOCKSTORAGEDRIVER_QEMU@" - -# Specifies cache-related options will be set to block devices or not. -# Default false -#block_device_cache_set = true - -# Specifies cache-related options for block devices. -# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. -# Default false -#block_device_cache_direct = true - -# Specifies cache-related options for block devices. -# Denotes whether flush requests for the device are ignored. -# Default false -#block_device_cache_noflush = true - -# Enable iothreads (data-plane) to be used. This causes IO to be -# handled in a separate IO thread. This is currently only implemented -# for SCSI. -# -enable_iothreads = @DEFENABLEIOTHREADS@ - -# Enable pre allocation of VM RAM, default false -# Enabling this will result in lower container density -# as all of the memory will be allocated and locked -# This is useful when you want to reserve all the memory -# upfront or in the cases where you want memory latencies -# to be very predictable -# Default false -#enable_mem_prealloc = true - -# Enable huge pages for VM RAM, default false -# Enabling this will result in the VM memory -# being allocated using huge pages. -# This is useful when you want to use vhost-user network -# stacks within the container. This will automatically -# result in memory pre allocation -#enable_hugepages = true - -# Enable vhost-user storage device, default false -# Enabling this will result in some Linux reserved block type -# major range 240-254 being chosen to represent vhost-user devices. -enable_vhost_user_store = @DEFENABLEVHOSTUSERSTORE@ - -# The base directory specifically used for vhost-user devices. -# Its sub-path "block" is used for block devices; "block/sockets" is -# where we expect vhost-user sockets to live; "block/devices" is where -# simulated block device nodes for vhost-user devices to live. -vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" - -# Enable vIOMMU, default false -# Enabling this will result in the VM having a vIOMMU device -# This will also add the following options to the kernel's -# command line: intel_iommu=on,iommu=pt -#enable_iommu = true - -# Enable IOMMU_PLATFORM, default false -# Enabling this will result in the VM device having iommu_platform=on set -#enable_iommu_platform = true - -# List of valid annotations values for the virtiofs daemon -# The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ -valid_vhost_user_store_paths = @DEFVALIDVHOSTUSERSTOREPATHS@ - -# Enable file based guest memory support. The default is an empty string which -# will disable this feature. In the case of virtio-fs, this is enabled -# automatically and '/dev/shm' is used as the backing folder. -# This option will be ignored if VM templating is enabled. -#file_mem_backend = "@DEFFILEMEMBACKEND@" - -# List of valid annotations values for the file_mem_backend annotation -# The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @DEFVALIDVHOSTUSERSTOREPATHS@ -valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@ - -# Enable swap of vm memory. Default false. -# The behaviour is undefined if mem_prealloc is also set to true -#enable_swap = true - -# This option changes the default hypervisor and kernel parameters -# to enable debug output where available. -# -# Default false -#enable_debug = true - -# Disable the customizations done in the runtime when it detects -# that it is running on top a VMM. This will result in the runtime -# behaving as it would when running on bare metal. -# -#disable_nesting_checks = true - -# This is the msize used for 9p shares. It is the number of bytes -# used for 9p packet payload. -#msize_9p = @DEFMSIZE9P@ - -# If false and nvdimm is supported, use nvdimm device to plug guest image. -# Otherwise virtio-block device is used. -# Default false -#disable_image_nvdimm = true - -# VFIO devices are hotplugged on a bridge by default. -# Enable hotplugging on root bus. This may be required for devices with -# a large PCI bar, as this is a current limitation with hotplugging on -# a bridge. This value is valid for "pc" machine type. -# Default false -#hotplug_vfio_on_root_bus = true - -# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off -# security (vhost-net runs ring0) for network I/O performance. -#disable_vhost_net = true - -# -# Default entropy source. -# The path to a host source of entropy (including a real hardware RNG) -# /dev/urandom and /dev/random are two main options. -# Be aware that /dev/random is a blocking source of entropy. If the host -# runs out of entropy, the VMs boot time will increase leading to get startup -# timeouts. -# The source of entropy /dev/urandom is non-blocking and provides a -# generally acceptable source of entropy. It should work well for pretty much -# all practical purposes. -#entropy_source= "@DEFENTROPYSOURCE@" - -# Path to OCI hook binaries in the *guest rootfs*. -# This does not affect host-side hooks which must instead be added to -# the OCI spec passed to the runtime. -# -# You can create a rootfs with hooks by customizing the osbuilder scripts: -# https://github.com/kata-containers/osbuilder -# -# Hooks must be stored in a subdirectory of guest_hook_path according to their -# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}". -# The agent will scan these directories for executable files and add them, in -# lexicographical order, to the lifecycle of the guest container. -# Hooks are executed in the runtime namespace of the guest. See the official documentation: -# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks -# Warnings will be logged if any error is encountered will scanning for hooks, -# but it will not abort container execution. -#guest_hook_path = "/usr/share/oci/hooks" - -[factory] -# VM templating support. Once enabled, new VMs are created from template -# using vm cloning. They will share the same initial kernel, initramfs and -# agent memory by mapping it readonly. It helps speeding up new container -# creation and saves a lot of memory if there are many kata containers running -# on the same host. -# -# When disabled, new VMs are created from scratch. -# -# Note: Requires "initrd=" to be set ("image=" is not supported). -# -# Default false -#enable_template = true - -# Specifies the path of template. -# -# Default "/run/vc/vm/template" -#template_path = "/run/vc/vm/template" - -# The number of caches of VMCache: -# unspecified or == 0 --> VMCache is disabled -# > 0 --> will be set to the specified number -# -# VMCache is a function that creates VMs as caches before using it. -# It helps speed up new container creation. -# The function consists of a server and some clients communicating -# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. -# The VMCache server will create some VMs and cache them by factory cache. -# It will convert the VM to gRPC format and transport it when gets -# requestion from clients. -# Factory grpccache is the VMCache client. It will request gRPC format -# VM and convert it back to a VM. If VMCache function is enabled, -# kata-runtime will request VM from factory grpccache when it creates -# a new sandbox. -# -# Default 0 -#vm_cache_number = 0 - -# Specify the address of the Unix socket that is used by VMCache. -# -# Default /var/run/kata-containers/cache.sock -#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" - -[agent.@PROJECT_TYPE@] -# If enabled, make the agent display debug-level messages. -# (default: disabled) -#enable_debug = true - -# Enable agent tracing. -# -# If enabled, the default trace mode is "dynamic" and the -# default trace type is "isolated". The trace mode and type are set -# explicity with the `trace_type=` and `trace_mode=` options. -# -# Notes: -# -# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly -# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` -# will NOT activate agent tracing. -# -# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for -# full details. -# -# (default: disabled) -#enable_tracing = true -# -#trace_mode = "dynamic" -#trace_type = "isolated" - -# Comma separated list of kernel modules and their parameters. -# These modules will be loaded in the guest kernel using modprobe(8). -# The following example can be used to load two kernel modules with parameters -# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] -# The first word is considered as the module name and the rest as its parameters. -# Container will not be started when: -# * A kernel module is specified and the modprobe command is not installed in the guest -# or it fails loading the module. -# * The module is not available in the guest or it doesn't met the guest kernel -# requirements, like architecture and version. -# -kernel_modules=[] - -# Enable debug console. - -# If enabled, user can connect guest OS running inside hypervisor -# through "kata-runtime exec " command - -#debug_console_enabled = true - -[netmon] -# If enabled, the network monitoring process gets started when the -# sandbox is created. This allows for the detection of some additional -# network being added to the existing network namespace, after the -# sandbox has been created. -# (default: disabled) -#enable_netmon = true - -# Specify the path to the netmon binary. -path = "@NETMONPATH@" - -# If enabled, netmon messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -[runtime] -# If enabled, the runtime will log additional debug messages to the -# system log -# (default: disabled) -#enable_debug = true -# -# Internetworking model -# Determines how the VM should be connected to the -# the container network interface -# Options: -# -# - bridged (Deprecated) -# Uses a linux bridge to interconnect the container interface to -# the VM. Works for most cases except macvlan and ipvlan. -# ***NOTE: This feature has been deprecated with plans to remove this -# feature in the future. Please use other network models listed below. -# -# - macvtap -# Used when the Container network interface can be bridged using -# macvtap. -# -# - none -# Used when customize network. Only creates a tap device. No veth pair. -# -# - tcfilter -# Uses tc filter rules to redirect traffic from the network interface -# provided by plugin to a tap interface connected to the VM. -# -internetworking_model="@DEFNETWORKMODEL_QEMU@" - -# disable guest seccomp -# Determines whether container seccomp profiles are passed to the virtual -# machine and applied by the kata agent. If set to true, seccomp is not applied -# within the guest -# (default: true) -disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ - -# If enabled, the runtime will create opentracing.io traces and spans. -# (See https://www.jaegertracing.io/docs/getting-started). -# (default: disabled) -#enable_tracing = true - -# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. -# This option may have some potential impacts to your host. It should only be used when you know what you're doing. -# `disable_new_netns` conflicts with `enable_netmon` -# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only -# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge -# (like OVS) directly. -# If you are using docker, `disable_new_netns` only works with `docker run --net=none` -# (default: false) -#disable_new_netns = true - -# if enabled, the runtime will add all the kata processes inside one dedicated cgroup. -# The container cgroups in the host are not created, just one single cgroup per sandbox. -# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. -# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. -# The sandbox cgroup is constrained if there is no container type annotation. -# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType -sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ - -# Enabled experimental feature list, format: ["a", "b"]. -# Experimental features are features not stable enough for production, -# they may break compatibility, and are prepared for a big version bump. -# Supported experimental features: -# (default: []) -experimental=@DEFAULTEXPFEATURES@ - -# If enabled, user can run pprof tools with shim v2 process through kata-monitor. -# (default: false) -# EnablePprof = true diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 0e6051430a..a5e4156043 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -122,7 +122,7 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # Shared file system type: # - virtio-9p (default) # - virtio-fs -shared_fs = "@DEFSHAREDFS@" +shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@" From f751c98da3453a1e6b2550e8ebeb44f231be1b59 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Tue, 13 Oct 2020 15:03:54 +0800 Subject: [PATCH 122/124] packaging: install virtiofsd for normal qemu build as well For experimental-virtiofs, we use it to test virtiofs with DAX. Let's rename its virtiofsd to virtiofsd-dax. Signed-off-by: Peng Tao --- src/runtime/Makefile | 3 +-- tools/packaging/scripts/configure-hypervisor.sh | 3 ++- tools/packaging/static-build/qemu-virtiofs/Dockerfile | 6 +++++- tools/packaging/static-build/qemu.blacklist | 2 +- tools/packaging/static-build/qemu/Dockerfile | 8 +++++++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 5a34b12139..690755f2ae 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -83,7 +83,6 @@ QEMUBINDIR := $(PREFIXDEPS)/bin CLHBINDIR := $(PREFIXDEPS)/bin FCBINDIR := $(PREFIXDEPS)/bin ACRNBINDIR := $(PREFIXDEPS)/bin -VIRTIOFSDBINDIR := $(PREFIXDEPS)/bin SYSCONFDIR := /etc LOCALSTATEDIR := /var @@ -184,7 +183,7 @@ DEFENTROPYSOURCE := /dev/urandom DEFDISABLEBLOCK := false DEFSHAREDFS := virtio-9p DEFSHAREDFS_QEMU_VIRTIOFS := virtio-fs -DEFVIRTIOFSDAEMON := $(VIRTIOFSDBINDIR)/virtiofsd +DEFVIRTIOFSDAEMON := $(LIBEXECDIR)/kata-qemu/virtiofsd DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"] # Default DAX mapping cache size in MiB #if value is 0, DAX is not enabled diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh index f8dd5f4c7a..195d25b3dc 100755 --- a/tools/packaging/scripts/configure-hypervisor.sh +++ b/tools/packaging/scripts/configure-hypervisor.sh @@ -262,7 +262,6 @@ generate_qemu_options() { qemu_options+=(size:--disable-snappy) # Disable unused security options - qemu_options+=(security:--disable-seccomp) qemu_options+=(security:--disable-tpm) # Disable userspace network access ("-net user") @@ -404,7 +403,9 @@ generate_qemu_options() { # operations safer. qemu_options+=(functionality:--enable-virtfs) qemu_options+=(functionality:--enable-attr) + # virtio-fs needs cap-ng and seccomp qemu_options+=(functionality:--enable-cap-ng) + qemu_options+=(functionality:--enable-seccomp) if [[ "${qemu_version_major}" -ge 4 || ( "${qemu_version_major}" -eq 3 && "${qemu_version_minor}" -ge 1 ) ]]; then # AVX2 is enabled by default by x86_64, make sure it's enabled only diff --git a/tools/packaging/static-build/qemu-virtiofs/Dockerfile b/tools/packaging/static-build/qemu-virtiofs/Dockerfile index 89c3e1abf5..c16049d3e9 100644 --- a/tools/packaging/static-build/qemu-virtiofs/Dockerfile +++ b/tools/packaging/static-build/qemu-virtiofs/Dockerfile @@ -1,3 +1,7 @@ +# Copyright (c) 2019 Intel Corporation +# Copyright (c) 2020 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 from ubuntu:20.04 ARG QEMU_VIRTIOFS_REPO @@ -68,5 +72,5 @@ RUN make -j$(nproc) RUN make -j$(nproc) virtiofsd RUN make install DESTDIR=/tmp/qemu-virtiofs-static RUN mv /tmp/qemu-virtiofs-static/"${PREFIX}"/bin/qemu-system-x86_64 /tmp/qemu-virtiofs-static/"${PREFIX}"/bin/qemu-virtiofs-system-x86_64 -RUN chmod +x virtiofsd && mv virtiofsd /tmp/qemu-virtiofs-static/opt/kata/bin/ +RUN mv /tmp/qemu-virtiofs-static/"${PREFIX}"/libexec/kata-qemu/virtiofsd /tmp/qemu-virtiofs-static/opt/kata/bin/virtiofsd-dax RUN cd /tmp/qemu-virtiofs-static && tar -czvf "${QEMU_TARBALL}" * diff --git a/tools/packaging/static-build/qemu.blacklist b/tools/packaging/static-build/qemu.blacklist index 1c2d9120db..e52c54dc9f 100644 --- a/tools/packaging/static-build/qemu.blacklist +++ b/tools/packaging/static-build/qemu.blacklist @@ -5,7 +5,7 @@ qemu_black_list=( */bin/qemu-pr-helper */bin/virtfs-proxy-helper -*/libexec/ +*/libexec/kata-qemu/qemu* */share/*/applications/ */share/*/*.dtb */share/*/efi-e1000e.rom diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index 4296e5e053..74d479c9fc 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -1,3 +1,7 @@ +# Copyright (c) 2019 Intel Corporation +# Copyright (c) 2020 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 from ubuntu:20.04 ARG QEMU_REPO @@ -36,7 +40,8 @@ RUN apt-get --no-install-recommends install -y \ libtool \ make \ pkg-config \ - pkg-config \ + libseccomp-dev \ + libseccomp2 \ python \ python-dev \ rsync \ @@ -56,5 +61,6 @@ RUN PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s kata-qemu | xargs ./conf --with-pkgversion=kata-static RUN make -j$(nproc) +RUN make -j$(nproc) virtiofsd RUN make install DESTDIR=/tmp/qemu-static RUN cd /tmp/qemu-static && tar -czvf "${QEMU_TARBALL}" * From 0c3b6a94b304d7501949d563b67909149a4b98d8 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Sat, 17 Oct 2020 11:27:09 +0800 Subject: [PATCH 123/124] package: drop qemu-virtiofs shim We have enabled qemu-virtiofs by default. Signed-off-by: Peng Tao --- .../packaging/kata-deploy/action/test-kata.sh | 2 +- .../nginx-deployment-qemu-virtiofs.yaml | 20 ------------------- .../kata-deploy/scripts/kata-deploy.sh | 1 - 3 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 tools/packaging/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml diff --git a/tools/packaging/kata-deploy/action/test-kata.sh b/tools/packaging/kata-deploy/action/test-kata.sh index 983e06417d..096ee9fa29 100755 --- a/tools/packaging/kata-deploy/action/test-kata.sh +++ b/tools/packaging/kata-deploy/action/test-kata.sh @@ -66,7 +66,7 @@ function run_test() { cmd="kubectl get pods | grep $busybox_pod | grep Completed" wait_time=120 - configurations=("nginx-deployment-qemu" "nginx-deployment-qemu-virtiofs" "nginx-deployment-clh") + configurations=("nginx-deployment-qemu" "nginx-deployment-clh") for deployment in "${configurations[@]}"; do # start the kata pod: kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml" diff --git a/tools/packaging/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml b/tools/packaging/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml deleted file mode 100644 index 06bf7d4497..0000000000 --- a/tools/packaging/kata-deploy/examples/nginx-deployment-qemu-virtiofs.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment-qemu-virtiofs -spec: - selector: - matchLabels: - app: nginx - replicas: 2 - template: - metadata: - labels: - app: nginx - spec: - runtimeClassName: kata-qemu-virtiofs - containers: - - name: nginx - image: nginx:1.14 - ports: - - containerPort: 80 diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index a4b49e7ab8..727098c2da 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -16,7 +16,6 @@ containerd_conf_file_backup="${containerd_conf_file}.bak" shims=( "fc" "qemu" - "qemu-virtiofs" "clh" ) From dbfe85e705e1247d2ea9ac77a4c1be84fe521cd3 Mon Sep 17 00:00:00 2001 From: Peng Tao Date: Sat, 17 Oct 2020 16:45:07 +0800 Subject: [PATCH 124/124] snap: install libseccomp-dev To build qemu with virtio-fs support. Depends-on: github.com/kata-containers/tests#2979 Fixes: #982 Signed-off-by: Peng Tao --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 37b2110f29..3b550b4763 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -284,7 +284,7 @@ parts: done # Only x86_64 supports libpmem - [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev + [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev libseccomp-dev configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh chmod +x ${configure_hypervisor}