From 44cde6e4642adba8999f6808118b293297a0a7e1 Mon Sep 17 00:00:00 2001 From: bin Date: Tue, 2 Feb 2021 14:13:34 +0800 Subject: [PATCH 1/4] runtime: connect guest debug console bypass kata-monitor Parse agent socket address by conversation to improve usability of using guest debug console. Fixes: #1329 Signed-off-by: bin --- src/runtime/cli/kata-exec.go | 35 ++++++++++++--------- src/runtime/pkg/kata-monitor/shim_client.go | 10 ++++-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/runtime/cli/kata-exec.go b/src/runtime/cli/kata-exec.go index 391cefe3d6..bbe8cd89e9 100644 --- a/src/runtime/cli/kata-exec.go +++ b/src/runtime/cli/kata-exec.go @@ -1,4 +1,5 @@ // Copyright (c) 2017-2019 Intel Corporation +// Copyright (c) 2020 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -19,6 +20,7 @@ import ( "time" "github.com/containerd/console" + kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" "github.com/pkg/errors" @@ -35,10 +37,10 @@ const ( subCommandName = "exec" // command-line parameters name - paramKataMonitorAddr = "kata-monitor-addr" + paramRuntimeNamespace = "runtime-namespace" paramDebugConsolePort = "kata-debug-port" defaultKernelParamDebugConsoleVPortValue = 1026 - defaultParamKataMonitorAddr = "http://localhost:8090" + defaultRuntimeNamespace = "k8s.io" ) var ( @@ -55,12 +57,12 @@ var kataExecCLICommand = cli.Command{ Usage: "Enter into guest by debug console", Flags: []cli.Flag{ cli.StringFlag{ - Name: paramKataMonitorAddr, - Usage: "Kata monitor listen address.", + Name: paramRuntimeNamespace, + Usage: "Namespace that containerd or CRI-O are using for containers. (Default: k8s.io, only works for containerd)", }, cli.Uint64Flag{ Name: paramDebugConsolePort, - Usage: "Port that debug console is listening on.", + Usage: "Port that debug console is listening on. (Default: 1026)", }, }, Action: func(context *cli.Context) error { @@ -71,11 +73,11 @@ var kataExecCLICommand = cli.Command{ span, _ := katautils.Trace(ctx, subCommandName) defer span.End() - endPoint := context.String(paramKataMonitorAddr) - if endPoint == "" { - endPoint = defaultParamKataMonitorAddr + namespace := context.String(paramRuntimeNamespace) + if namespace == "" { + namespace = defaultRuntimeNamespace } - span.SetAttributes(label.Key("endPoint").String(endPoint)) + span.SetAttributes(label.Key("namespace").String(namespace)) port := context.Uint64(paramDebugConsolePort) if port == 0 { @@ -89,7 +91,7 @@ var kataExecCLICommand = cli.Command{ } span.SetAttributes(label.Key("sandbox").String(sandboxID)) - conn, err := getConn(endPoint, sandboxID, port) + conn, err := getConn(namespace, sandboxID, port) if err != nil { return err } @@ -172,15 +174,20 @@ func (s *iostream) Read(data []byte) (n int, err error) { return s.conn.Read(data) } -func getConn(endPoint, sandboxID string, port uint64) (net.Conn, error) { - shimURL := fmt.Sprintf("%s/agent-url?sandbox=%s", endPoint, sandboxID) - resp, err := http.Get(shimURL) +func getConn(namespace, sandboxID string, port uint64) (net.Conn, error) { + socketAddr := fmt.Sprintf("/containerd-shim/%s/%s/shim-monitor.sock", namespace, sandboxID) + client, err := kataMonitor.BuildUnixSocketClient(socketAddr, defaultTimeout) + if err != nil { + return nil, err + } + + resp, err := client.Get("http://shim/agent-url") if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Failed to get %s: %d", shimURL, resp.StatusCode) + return nil, fmt.Errorf("Failed to get %s: %d", socketAddr, resp.StatusCode) } defer resp.Body.Close() diff --git a/src/runtime/pkg/kata-monitor/shim_client.go b/src/runtime/pkg/kata-monitor/shim_client.go index 0c1c1c81a7..f711f88f99 100644 --- a/src/runtime/pkg/kata-monitor/shim_client.go +++ b/src/runtime/pkg/kata-monitor/shim_client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Ant Financial +// Copyright (c) 2020-2021 Ant Group // // SPDX-License-Identifier: Apache-2.0 // @@ -34,15 +34,19 @@ func getSandboxIdFromReq(r *http.Request) (string, error) { } func (km *KataMonitor) buildShimClient(sandboxID, namespace string, timeout time.Duration) (*http.Client, error) { - socket, err := km.getMonitorAddress(sandboxID, namespace) + socketAddr, err := km.getMonitorAddress(sandboxID, namespace) if err != nil { return nil, err } + return BuildUnixSocketClient(socketAddr, timeout) +} +// BuildUnixSocketClient build http client for Unix socket +func BuildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) { transport := &http.Transport{ DisableKeepAlives: true, Dial: func(proto, addr string) (conn net.Conn, err error) { - return net.Dial("unix", "\x00"+socket) + return net.Dial("unix", "\x00"+socketAddr) }, } From 9963428a4dfc36a19d97f5353fec8d7f8bf351b0 Mon Sep 17 00:00:00 2001 From: bin Date: Tue, 2 Feb 2021 14:22:24 +0800 Subject: [PATCH 2/4] docs: update document for using debug console Delete using `kata-monitor` to use `kata-runtime exec` Fixes: #1329 Signed-off-by: bin --- docs/Developer-Guide.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index d82de38fee..ab4ae1247c 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -37,7 +37,6 @@ * [Set up a debug console](#set-up-a-debug-console) * [Simple debug console setup](#simple-debug-console-setup) * [Enable agent debug console](#enable-agent-debug-console) - * [Start `kata-monitor`](#start-kata-monitor) * [Connect to debug console](#connect-to-debug-console) * [Traditional debug console setup](#traditional-debug-console-setup) * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) @@ -477,17 +476,6 @@ debug_console_enabled = true This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as kernel parameters, and sandboxes created using this parameters will start a shell in guest if new connection is accept from VSOCK. -#### Start `kata-monitor` - -The `kata-runtime exec` command needs `kata-monitor` to get the sandbox's `vsock` address to connect to, first start `kata-monitor`. - -``` -$ sudo kata-monitor -``` - -`kata-monitor` will serve at `localhost:8090` by default. - - #### Connect to debug console Command `kata-runtime exec` is used to connect to the debug console. @@ -502,6 +490,8 @@ bash-4.2# exit exit ``` +`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which namespace the containers(Pods) are created. By default, it is set to `k8s.io` and works for containerd, for other upper runtimes, you may need to set the namespace by `runtime-namespace` option. + If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup). ### Traditional debug console setup From f4ae9c84768f60a0a8774d6cb26beffb04ac1e74 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Thu, 4 Feb 2021 10:22:01 +0800 Subject: [PATCH 3/4] docs: Update Developer-Guide.md Add description for difference of namespace in containerd and Kuberenetes. Co-authored-by: Eric Ernst Signed-off-by: bin --- docs/Developer-Guide.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index ab4ae1247c..d6f71af065 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -490,7 +490,9 @@ bash-4.2# exit exit ``` -`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which namespace the containers(Pods) are created. By default, it is set to `k8s.io` and works for containerd, for other upper runtimes, you may need to set the namespace by `runtime-namespace` option. +`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured + with Kubernetes. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). +For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option. If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup). From 10f1c30f3d9162df7537e86f6bdfac48bdf96161 Mon Sep 17 00:00:00 2001 From: bin Date: Thu, 4 Feb 2021 21:57:52 +0800 Subject: [PATCH 4/4] kata-runtime: use filepath.Join() to compose file path Use filepath.Join() will be safer than format string directly. Signed-off-by: bin --- docs/Developer-Guide.md | 2 +- src/runtime/cli/kata-exec.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index d6f71af065..7dd6fed24c 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -491,7 +491,7 @@ exit ``` `kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured - with Kubernetes. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). + with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option. If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup). diff --git a/src/runtime/cli/kata-exec.go b/src/runtime/cli/kata-exec.go index bbe8cd89e9..83a314aa72 100644 --- a/src/runtime/cli/kata-exec.go +++ b/src/runtime/cli/kata-exec.go @@ -14,6 +14,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "sync" @@ -175,7 +176,7 @@ func (s *iostream) Read(data []byte) (n int, err error) { } func getConn(namespace, sandboxID string, port uint64) (net.Conn, error) { - socketAddr := fmt.Sprintf("/containerd-shim/%s/%s/shim-monitor.sock", namespace, sandboxID) + socketAddr := filepath.Join(string(filepath.Separator), "containerd-shim", namespace, sandboxID, "shim-monitor.sock") client, err := kataMonitor.BuildUnixSocketClient(socketAddr, defaultTimeout) if err != nil { return nil, err