From febdf8f68c2b6a3a1c8d13d589c668ef2fd5ee1c Mon Sep 17 00:00:00 2001 From: bin liu Date: Thu, 16 Jul 2020 16:13:05 +0800 Subject: [PATCH 1/2] runtime: add debug console service Add `kata-runtime exec` to enter guest OS through shell started by agent Fixes: #245 Signed-off-by: bin liu --- docs/Developer-Guide.md | 51 ++++- src/agent/src/main.rs | 208 ++++++++++++++++-- src/runtime/cli/config-generated.go.in | 1 + src/runtime/cli/kata-exec.go | 208 ++++++++++++++++++ src/runtime/cli/kata-monitor/main.go | 1 + src/runtime/cli/main.go | 1 + .../containerd-shim-v2/shim_management.go | 14 ++ src/runtime/pkg/kata-monitor/metrics.go | 29 +-- src/runtime/pkg/kata-monitor/monitor.go | 22 ++ src/runtime/pkg/kata-monitor/pprof.go | 8 - src/runtime/pkg/kata-monitor/shim_client.go | 81 +++++++ src/runtime/virtcontainers/interfaces.go | 1 + .../pkg/agent/protocols/client/client.go | 4 +- .../pkg/annotations/annotations.go | 2 +- .../virtcontainers/pkg/vcmock/sandbox.go | 7 + .../virtcontainers/pkg/vcmock/types.go | 1 + src/runtime/virtcontainers/sandbox.go | 4 + 17 files changed, 578 insertions(+), 65 deletions(-) create mode 100644 src/runtime/cli/kata-exec.go create mode 100644 src/runtime/pkg/kata-monitor/shim_client.go diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 7ff1ef9463..c5cb3583e7 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -34,7 +34,11 @@ * [Troubleshoot Kata Containers](#troubleshoot-kata-containers) * [Appendices](#appendices) * [Checking Docker default runtime](#checking-docker-default-runtime) - * [Set up a debug console](#set-up-a-debug-console) + * [Set up a debug console(the easy way)](#set-up-a-debug-consolethe-easy-way) + * [Enable agent debug console](#enable-agent-debug-console) + * [Start `kata-monitor`](#start-kata-monitor) + * [Connect to debug console](#connect-to-debug-console) + * [Set up a debug console(the traditional way)](#set-up-a-debug-consolethe-traditional-way) * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) * [Create a debug systemd service](#create-a-debug-systemd-service) * [Build the debug image](#build-the-debug-image) @@ -60,7 +64,7 @@ The recommended way to create a development environment is to first to create a working system. The installation guide instructions will install all required Kata Containers -components, plus Docker*, the hypervisor, and the Kata Containers image and +components, plus *Docker*, the hypervisor, and the Kata Containers image and guest kernel. # Requirements to build individual components @@ -434,7 +438,48 @@ See [Set up a debug console](#set-up-a-debug-console). $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime" ``` -## Set up a debug console +## Set up a debug console(The easy way) + +Kata containers 2.0 support a shell simulated *console* for quickly debug purpose. This approach use `vsock` to connect shell running inside guest started by agent. The good aspect is that we need not modify guest image or despite using what device that hypervisors support. Only `/bin/sh` or `/bin/bash` are necessary. + +### Enable agent debug console + +Change your `configuration.toml`, add agent debug parameters. + +``` +kernel_params = "agent.debug_console agent.debug_console_vport=1026" +``` + +Sandboxes created using this parameters will start a shell in guest if new connection is accept from `vsock`. + +### Start `kata-monitor` + +`kata-runitime exec` need `kata-monitor` to get the sandbox's `vsock` address to connect to, firt start `kata-monitor`. + +``` +$ sudo kata-monitor +``` + +`kata-monitor` will serve at `localhost:8090` by default. + + +### Connect to debug console + +Command `kata-runitime exec` is used to connect to the debug console. + +``` +$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd +bash-4.2# id +uid=0(root) gid=0(root) groups=0(root) +bash-4.2# pwd +/ +bash-4.2# exit +exit +``` + +If you want to access guest OS through a traditional way, see [Set up a debug console(the traditional way)](#set-up-a-debug-console-the-traditional-way). + +## Set up a debug console(the traditional way) By default you cannot login to a virtual machine, since this can be sensitive from a security perspective. Also, allowing logins would require additional diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 25b6218b51..138527f247 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -31,16 +31,20 @@ extern crate netlink; use crate::netlink::{RtnlHandle, NETLINK_ROUTE}; use anyhow::{anyhow, Context, Result}; use nix::fcntl::{self, OFlag}; +use nix::fcntl::{FcntlArg, FdFlag}; +use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use nix::pty; +use nix::sys::select::{select, FdSet}; use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType}; use nix::sys::wait::{self, WaitStatus}; -use nix::unistd; -use nix::unistd::dup; +use nix::unistd::{self, close, dup, dup2, fork, setsid, ForkResult}; use prctl::set_child_subreaper; use signal_hook::{iterator::Signals, SIGCHLD}; use std::collections::HashMap; use std::env; -use std::ffi::OsStr; +use std::ffi::{CStr, CString, OsStr}; use std::fs::{self, File}; +use std::io::{Read, Write}; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs as unixfs; use std::os::unix::io::AsRawFd; @@ -75,6 +79,8 @@ const NAME: &str = "kata-agent"; const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline"; const CONSOLE_PATH: &str = "/dev/console"; +const DEFAULT_BUF_SIZE: usize = 8 * 1024; + lazy_static! { static ref GLOBAL_DEVICE_WATCHER: Arc>>> = Arc::new(Mutex::new(HashMap::new())); @@ -213,7 +219,7 @@ fn start_sandbox(logger: &Logger, config: &agentConfig, init_mode: bool) -> Resu let handle = builder.spawn(move || { let shells = shells.lock().unwrap(); - let result = setup_debug_console(shells.to_vec(), debug_console_vport); + let result = setup_debug_console(&thread_logger, shells.to_vec(), debug_console_vport); if result.is_err() { // Report error, but don't fail warn!(thread_logger, "failed to setup debug console"; @@ -406,9 +412,9 @@ use crate::config::agentConfig; use nix::sys::stat::Mode; use std::os::unix::io::{FromRawFd, RawFd}; use std::path::PathBuf; -use std::process::{exit, Command, Stdio}; +use std::process::exit; -fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { +fn setup_debug_console(logger: &Logger, shells: Vec, port: u32) -> Result<()> { let mut shell: &str = ""; for sh in shells.iter() { let binary = PathBuf::from(sh); @@ -422,7 +428,7 @@ fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { return Err(anyhow!("no shell found to launch debug console")); } - let f: RawFd = if port > 0 { + if port > 0 { let listenfd = socket::socket( AddressFamily::Vsock, SockType::Stream, @@ -432,29 +438,183 @@ fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port); socket::bind(listenfd, &addr)?; socket::listen(listenfd, 1)?; - socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)? + loop { + let f: RawFd = socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?; + match run_debug_console_shell(logger, shell, f) { + Ok(_) => { + info!(logger, "run_debug_console_shell session finished"); + } + Err(err) => { + error!(logger, "run_debug_console_shell failed: {:?}", err); + } + } + } } else { let mut flags = OFlag::empty(); flags.insert(OFlag::O_RDWR); flags.insert(OFlag::O_CLOEXEC); - fcntl::open(CONSOLE_PATH, flags, Mode::empty())? + loop { + let f: RawFd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?; + match run_debug_console_shell(logger, shell, f) { + Ok(_) => { + info!(logger, "run_debug_console_shell session finished"); + } + Err(err) => { + error!(logger, "run_debug_console_shell failed: {:?}", err); + } + } + } + }; +} + +fn io_copy(reader: &mut R, writer: &mut W) -> io::Result +where + R: Read, + W: Write, +{ + let mut buf = [0; DEFAULT_BUF_SIZE]; + let buf_len; + + match reader.read(&mut buf) { + Ok(0) => return Ok(0), + Ok(len) => buf_len = len, + Err(err) => return Err(err), }; - let cmd = Command::new(shell) - .arg("-i") - .stdin(unsafe { Stdio::from_raw_fd(f) }) - .stdout(unsafe { Stdio::from_raw_fd(f) }) - .stderr(unsafe { Stdio::from_raw_fd(f) }) - .spawn(); + // write and return + match writer.write_all(&buf[..buf_len]) { + Ok(_) => return Ok(buf_len as u64), + Err(err) => return Err(err), + } +} - let mut cmd = match cmd { - Ok(c) => c, - Err(_) => return Err(anyhow!("failed to spawn shell")), - }; +fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Result<()> { + let pseduo = pty::openpty(None, None)?; + let _ = fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + let _ = fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - cmd.wait()?; + let slave_fd = pseduo.slave; - return Ok(()); + match fork() { + Ok(ForkResult::Child) => { + // create new session with child as session leader + setsid()?; + + // dup stdin, stdout, stderr to let child act as a terminal + dup2(slave_fd, STDIN_FILENO)?; + dup2(slave_fd, STDOUT_FILENO)?; + dup2(slave_fd, STDERR_FILENO)?; + + // set tty + unsafe { + libc::ioctl(0, libc::TIOCSCTTY); + } + + let cmd = CString::new(shell).unwrap(); + let args: Vec<&CStr> = vec![]; + + // run shell + if let Err(e) = unistd::execvp(cmd.as_c_str(), args.as_slice()) { + match e { + nix::Error::Sys(errno) => { + std::process::exit(errno as i32); + } + _ => std::process::exit(-2), + } + } + } + + Ok(ForkResult::Parent { child: child_pid }) => { + info!(logger, "get debug shell pid {:?}", child_pid); + + let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?; + let master_fd = pseduo.master; + let debug_shell_logger = logger.clone(); + + // start a thread to do IO copy between socket and pseduo.master + thread::spawn(move || { + let mut master_reader = unsafe { File::from_raw_fd(master_fd) }; + let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) }; + let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) }; + let mut master_writer = unsafe { File::from_raw_fd(master_fd) }; + + loop { + let mut fd_set = FdSet::new(); + fd_set.insert(master_fd); + fd_set.insert(socket_fd); + fd_set.insert(rfd); + + match select( + Some(fd_set.highest().unwrap() + 1), + &mut fd_set, + None, + None, + None, + ) { + Ok(_) => (), + Err(e) => { + if e == nix::Error::from(nix::errno::Errno::EINTR) { + continue; + } else { + error!(debug_shell_logger, "select error {:?}", e); + break; + } + } + } + + if fd_set.contains(master_fd) { + match io_copy(&mut master_reader, &mut socket_writer) { + Ok(0) => { + debug!(debug_shell_logger, "master fd closed"); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read master fd error {:?}", e); + break; + } + } + } + + if fd_set.contains(socket_fd) { + match io_copy(&mut socket_reader, &mut master_writer) { + Ok(0) => { + debug!(debug_shell_logger, "master fd closed"); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read master fd error {:?}", e); + break; + } + } + } + + if fd_set.contains(rfd) { + info!( + debug_shell_logger, + "debug shelll process {} exited", child_pid + ); + break; + } + } + }); + + let wait_status = wait::waitpid(child_pid, None); + info!(logger, "debug console exit code: {:?}", wait_status); + + // close pipe to exit select loop + let _ = close(wfd); + } + Err(err) => { + let msg = format!("fork error: {:?}", err); + return Err(ErrorKind::ErrorCode(msg).into()); + } + } + + Ok(()) } #[cfg(test)] @@ -472,8 +632,9 @@ mod tests { let shells_ref = SHELLS.clone(); let mut shells = shells_ref.lock().unwrap(); shells.clear(); + let logger = slog_scope::logger(); - let result = setup_debug_console(shells.to_vec(), 0); + let result = setup_debug_console(&logger, shells.to_vec(), 0); assert!(result.is_err()); assert_eq!( @@ -498,8 +659,9 @@ mod tests { .to_string(); shells.push(shell); + let logger = slog_scope::logger(); - let result = setup_debug_console(shells.to_vec(), 0); + let result = setup_debug_console(&logger, shells.to_vec(), 0); assert!(result.is_err()); assert_eq!( diff --git a/src/runtime/cli/config-generated.go.in b/src/runtime/cli/config-generated.go.in index 41ce38ac3b..fc69f13660 100644 --- a/src/runtime/cli/config-generated.go.in +++ b/src/runtime/cli/config-generated.go.in @@ -39,6 +39,7 @@ var version = "@VERSION@" // project-specific command names var envCmd = fmt.Sprintf("%s-env", projectPrefix) var checkCmd = fmt.Sprintf("%s-check", projectPrefix) +var execCmd = "exec" // project-specific option names var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix) diff --git a/src/runtime/cli/kata-exec.go b/src/runtime/cli/kata-exec.go new file mode 100644 index 0000000000..ed3a7cabd5 --- /dev/null +++ b/src/runtime/cli/kata-exec.go @@ -0,0 +1,208 @@ +// Copyright (c) 2017-2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strings" + + "sync" + "time" + + "github.com/containerd/console" + "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +const ( + + // The buffer size used to specify the buffer for IO streams copy + bufSize = 32 << 10 + + defaultTimeout = 3 * time.Second +) + +var ( + bufPool = sync.Pool{ + New: func() interface{} { + buffer := make([]byte, bufSize) + return &buffer + }, + } +) + +var kataExecCLICommand = cli.Command{ + Name: execCmd, + Usage: "Enter into guest by debug console", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "monitor-addr", + Usage: "Kata monitor listen address.", + }, + cli.Uint64Flag{ + Name: "debug-port", + Usage: "Port that debug console is listening on.", + }, + }, + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { + return err + } + span, _ := katautils.Trace(ctx, "exec") + defer span.Finish() + + endPoint := context.String("monitor-addr") + if endPoint == "" { + endPoint = "http://localhost:8090" + } + + port := context.Uint64("debug-port") + if port == 0 { + port = 1026 + } + + sandboxID := context.Args().Get(0) + if sandboxID == "" { + return fmt.Errorf("SandboxID not found") + } + + conn, err := getConn(endPoint, sandboxID, port) + if err != nil { + return err + } + defer conn.Close() + + con := console.Current() + defer con.Reset() + + if err := con.SetRaw(); err != nil { + return err + } + + iostream := &iostream{ + conn: conn, + exitch: make(chan struct{}), + closed: false, + } + + ioCopy(iostream, con) + + <-iostream.exitch + return nil + }, +} + +func ioCopy(stream *iostream, con console.Console) { + var wg sync.WaitGroup + + // stdin + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + io.CopyBuffer(stream, con, *p) + }() + + // stdout + wg.Add(1) + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + io.CopyBuffer(os.Stdout, stream, *p) + wg.Done() + }() + + wg.Wait() + close(stream.exitch) +} + +type iostream struct { + conn net.Conn + exitch chan struct{} + closed bool +} + +func (s *iostream) Write(data []byte) (n int, err error) { + if s.closed { + return 0, errors.New("stream closed") + } + return s.conn.Write(data) +} + +func (s *iostream) Close() error { + if s.closed { + return errors.New("stream closed") + } + + err := s.conn.Close() + if err == nil { + s.closed = true + } + + return err +} + +func (s *iostream) Read(data []byte) (n int, err error) { + if s.closed { + return 0, errors.New("stream closed") + } + + return s.conn.Read(data) +} + +func getConn(endPoint, sandboxID string, port uint64) (net.Conn, error) { + shimURL := fmt.Sprintf("%s/agent-url?sandbox=%s", endPoint, sandboxID) + resp, err := http.Get(shimURL) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Failed to get %s: %d", shimURL, resp.StatusCode) + } + + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + sock := strings.TrimSuffix(string(data), "\n") + addr, err := url.Parse(sock) + if err != nil { + return nil, err + } + + // validate more + switch addr.Scheme { + case clientUtils.VSockSocketScheme: + // vsock://31513974:1024 + shimAddr := clientUtils.VSockSocketScheme + ":" + addr.Host + shimAddr = strings.Replace(shimAddr, ":1024", fmt.Sprintf(":%d", port), -1) + return clientUtils.VsockDialer(shimAddr, defaultTimeout) + + case clientUtils.HybridVSockScheme: + // addr: hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock:1024 + hvsocket := strings.Split(addr.Path, ":") + if len(hvsocket) != 2 { + return nil, fmt.Errorf("Invalid hybrid vsock scheme: %s", sock) + } + + // hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock + shimAddr := fmt.Sprintf("%s:%s:%d", clientUtils.HybridVSockScheme, hvsocket[0], port) + return clientUtils.HybridVSockDialer(shimAddr, defaultTimeout) + } + + return nil, fmt.Errorf("schema %s not found", addr.Scheme) +} diff --git a/src/runtime/cli/kata-monitor/main.go b/src/runtime/cli/kata-monitor/main.go index 1b5920361e..6f66504b03 100644 --- a/src/runtime/cli/kata-monitor/main.go +++ b/src/runtime/cli/kata-monitor/main.go @@ -36,6 +36,7 @@ func main() { m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(km.ProcessMetricsRequest)) m.Handle("/sandboxes", http.HandlerFunc(km.ListSandboxes)) + m.Handle("/agent-url", http.HandlerFunc(km.GetAgentURL)) // for debug shim process m.Handle("/debug/vars", http.HandlerFunc(km.ExpvarHandler)) diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go index a8b3c42c2a..2b721eaa21 100644 --- a/src/runtime/cli/main.go +++ b/src/runtime/cli/main.go @@ -125,6 +125,7 @@ var runtimeCommands = []cli.Command{ // Kata Containers specific extensions kataCheckCLICommand, kataEnvCLICommand, + kataExecCLICommand, factoryCLICommand, } diff --git a/src/runtime/containerd-shim-v2/shim_management.go b/src/runtime/containerd-shim-v2/shim_management.go index 9afaf3b4b9..f971a2a602 100644 --- a/src/runtime/containerd-shim-v2/shim_management.go +++ b/src/runtime/containerd-shim-v2/shim_management.go @@ -8,6 +8,7 @@ package containerdshim import ( "context" "expvar" + "fmt" "io" "net/http" "net/http/pprof" @@ -34,6 +35,18 @@ var ( shimMgtLog = shimLog.WithField("subsystem", "shim-management") ) +// agentURL returns URL for agent +func (s *service) agentURL(w http.ResponseWriter, r *http.Request) { + url, err := s.sandbox.GetAgentURL() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + fmt.Fprint(w, url) +} + // serveMetrics handle /metrics requests func (s *service) serveMetrics(w http.ResponseWriter, r *http.Request) { @@ -139,6 +152,7 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec // bind hanlder m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) + m.Handle("/agent-url", http.HandlerFunc(s.agentURL)) s.mountPprofHandle(m, ociSpec) // register shim metrics diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index 712924ffed..b4fcd5830a 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -10,7 +10,6 @@ import ( "compress/gzip" "io" "io/ioutil" - "net" "net/http" "path/filepath" "sort" @@ -236,33 +235,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { // getSandboxMetrics will get sandbox's metrics from shim func (km *KataMonitor) getSandboxMetrics(sandboxID, namespace string) ([]*dto.MetricFamily, error) { - socket, err := km.getMonitorAddress(sandboxID, namespace) - if err != nil { - return nil, err - } - - transport := &http.Transport{ - DisableKeepAlives: true, - Dial: func(proto, addr string) (conn net.Conn, err error) { - return net.Dial("unix", "\x00"+socket) - }, - } - - client := http.Client{ - Timeout: 3 * time.Second, - Transport: transport, - } - - resp, err := client.Get("http://shim/metrics") - if err != nil { - return nil, err - } - - defer func() { - resp.Body.Close() - }() - - body, err := ioutil.ReadAll(resp.Body) + body, err := km.doGet(sandboxID, namespace, defaultTimeout, "metrics") if err != nil { return nil, err } diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index 7f4cb4eeaf..3254b52021 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -80,6 +80,28 @@ func (km *KataMonitor) initSandboxCache() error { return nil } +// GetAgentURL returns agent URL +func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) { + sandboxID, err := getSandboxIdFromReq(r) + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + namespace, err := km.getSandboxNamespace(sandboxID) + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + + data, err := km.doGet(sandboxID, namespace, defaultTimeout, "agent-url") + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + + fmt.Fprintln(w, string(data)) +} + // ListSandboxes list all sandboxes running in Kata func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) { sandboxes := km.getSandboxList() diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go index e043251cb7..9e54315a42 100644 --- a/src/runtime/pkg/kata-monitor/pprof.go +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -12,14 +12,6 @@ import ( "net/http" ) -func getSandboxIdFromReq(r *http.Request) (string, error) { - sandbox := r.URL.Query().Get("sandbox") - if sandbox != "" { - return sandbox, nil - } - return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query()) -} - func serveError(w http.ResponseWriter, status int, txt string) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("X-Go-Pprof", "1") diff --git a/src/runtime/pkg/kata-monitor/shim_client.go b/src/runtime/pkg/kata-monitor/shim_client.go new file mode 100644 index 0000000000..0c1c1c81a7 --- /dev/null +++ b/src/runtime/pkg/kata-monitor/shim_client.go @@ -0,0 +1,81 @@ +// Copyright (c) 2020 Ant Financial +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "time" +) + +const ( + defaultTimeout = 3 * time.Second +) + +func commonServeError(w http.ResponseWriter, status int, err error) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(status) + if err != nil { + fmt.Fprintln(w, err.Error()) + } +} + +func getSandboxIdFromReq(r *http.Request) (string, error) { + sandbox := r.URL.Query().Get("sandbox") + if sandbox != "" { + return sandbox, nil + } + return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query()) +} + +func (km *KataMonitor) buildShimClient(sandboxID, namespace string, timeout time.Duration) (*http.Client, error) { + socket, err := km.getMonitorAddress(sandboxID, namespace) + if err != nil { + return nil, err + } + + transport := &http.Transport{ + DisableKeepAlives: true, + Dial: func(proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", "\x00"+socket) + }, + } + + client := &http.Client{ + Transport: transport, + } + + if timeout > 0 { + client.Timeout = timeout + } + + return client, nil +} + +func (km *KataMonitor) doGet(sandboxID, namespace string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { + client, err := km.buildShimClient(sandboxID, namespace, timeoutInSeconds) + if err != nil { + return nil, err + } + + resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) + if err != nil { + return nil, err + } + + defer func() { + resp.Body.Close() + }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/src/runtime/virtcontainers/interfaces.go b/src/runtime/virtcontainers/interfaces.go index c04acb676e..365c329db0 100644 --- a/src/runtime/virtcontainers/interfaces.go +++ b/src/runtime/virtcontainers/interfaces.go @@ -75,6 +75,7 @@ type VCSandbox interface { UpdateRuntimeMetrics() error GetAgentMetrics() (string, error) + GetAgentURL() (string, error) } // VCContainer is the Container interface diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go index 200f4584b6..c3b1c9390b 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go @@ -178,7 +178,7 @@ func parse(sock string) (string, *url.URL, error) { func agentDialer(addr *url.URL) dialer { switch addr.Scheme { case VSockSocketScheme: - return vsockDialer + return VsockDialer case HybridVSockScheme: return HybridVSockDialer case MockHybridVSockScheme: @@ -278,7 +278,7 @@ func commonDialer(timeout time.Duration, dialFunc func() (net.Conn, error), time return conn, nil } -func vsockDialer(sock string, timeout time.Duration) (net.Conn, error) { +func VsockDialer(sock string, timeout time.Duration) (net.Conn, error) { cid, port, err := parseGrpcVsockAddr(sock) if err != nil { return nil, err diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index c84d9d594e..b9b3bf2fa3 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -245,7 +245,7 @@ const ( // The following example can be used to load two kernel modules with parameters /// // annotations: - // io.kata-containers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" + // io.katacontainers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" // // The first word is considered as the module name and the rest as its parameters. // diff --git a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go index a7d4c6b7e4..727fed7af8 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go +++ b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go @@ -247,3 +247,10 @@ func (s *Sandbox) Stats() (vc.SandboxStats, error) { } return vc.SandboxStats{}, nil } + +func (s *Sandbox) GetAgentURL() (string, error) { + if s.GetAgentURLFunc != nil { + return s.GetAgentURLFunc() + } + return "", nil +} diff --git a/src/runtime/virtcontainers/pkg/vcmock/types.go b/src/runtime/virtcontainers/pkg/vcmock/types.go index 39ea6fc4f5..211d97c8e2 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/types.go +++ b/src/runtime/virtcontainers/pkg/vcmock/types.go @@ -67,6 +67,7 @@ type Sandbox struct { UpdateRuntimeMetricsFunc func() error GetAgentMetricsFunc func() (string, error) StatsFunc func() (vc.SandboxStats, error) + GetAgentURLFunc func() (string, error) } // Container is a fake Container type used for testing diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index f588d472b7..92a7917f95 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2270,3 +2270,7 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec { func (s *Sandbox) GetOOMEvent() (string, error) { return s.agent.getOOMEvent() } + +func (s *Sandbox) GetAgentURL() (string, error) { + return s.agent.getAgentURL() +} From 484a595f1a4633d0aa9bb901f45b662562b34b2b Mon Sep 17 00:00:00 2001 From: bin liu Date: Mon, 27 Jul 2020 13:09:34 +0800 Subject: [PATCH 2/2] runtime: add enable_debug_console configuration item for agent Set enable_debug_console=true in Kata's congiguration file, runtime will pass `agent.debug_console` and `agent.debug_console_vport=1026` to agent. Fixes: #245 Signed-off-by: bin liu --- docs/Developer-Guide.md | 71 +++++++------- src/agent/src/main.rs | 92 +++++++++++-------- src/runtime/cli/config-generated.go.in | 1 - .../cli/config/configuration-acrn.toml.in | 7 ++ .../cli/config/configuration-clh.toml.in | 6 ++ .../cli/config/configuration-fc.toml.in | 7 ++ .../configuration-qemu-virtiofs.toml.in | 6 ++ .../cli/config/configuration-qemu.toml.in | 6 ++ src/runtime/cli/kata-exec.go | 32 ++++--- src/runtime/pkg/katautils/config.go | 41 ++++----- src/runtime/pkg/katautils/config_test.go | 13 ++- src/runtime/virtcontainers/kata_agent.go | 35 +++---- src/runtime/virtcontainers/vm_test.go | 10 +- 13 files changed, 202 insertions(+), 125 deletions(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index c5cb3583e7..b66f02b1ee 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -34,19 +34,20 @@ * [Troubleshoot Kata Containers](#troubleshoot-kata-containers) * [Appendices](#appendices) * [Checking Docker default runtime](#checking-docker-default-runtime) - * [Set up a debug console(the easy way)](#set-up-a-debug-consolethe-easy-way) - * [Enable agent debug console](#enable-agent-debug-console) - * [Start `kata-monitor`](#start-kata-monitor) - * [Connect to debug console](#connect-to-debug-console) - * [Set up a debug console(the traditional way)](#set-up-a-debug-consolethe-traditional-way) - * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) - * [Create a debug systemd service](#create-a-debug-systemd-service) - * [Build the debug image](#build-the-debug-image) - * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image) - * [Ensure debug options are valid](#ensure-debug-options-are-valid) - * [Create a container](#create-a-container) - * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) - * [Obtain details of the image](#obtain-details-of-the-image) + * [Set up a debug console](#set-up-a-debug-console) + * [Simple debug console setup](#simple-debug-console-setup) + * [Enable agent debug console](#enable-agent-debug-console) + * [Start `kata-monitor`](#start-kata-monitor) + * [Connect to debug console](#connect-to-debug-console) + * [Traditional debug console setup](#traditional-simple-debug-console-setup) + * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) + * [Create a debug systemd service](#create-a-debug-systemd-service) + * [Build the debug image](#build-the-debug-image) + * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image) + * [Ensure debug options are valid](#ensure-debug-options-are-valid) + * [Create a container](#create-a-container) + * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) + * [Obtain details of the image](#obtain-details-of-the-image) * [Capturing kernel boot logs](#capturing-kernel-boot-logs) # Warning @@ -437,24 +438,30 @@ See [Set up a debug console](#set-up-a-debug-console). ``` $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime" ``` +## Set up a debug console -## Set up a debug console(The easy way) +Kata containers provides two ways to connect to the guest. One is using traditional login service, which needs additional works. In contrast the simple debug console is easy to setup. -Kata containers 2.0 support a shell simulated *console* for quickly debug purpose. This approach use `vsock` to connect shell running inside guest started by agent. The good aspect is that we need not modify guest image or despite using what device that hypervisors support. Only `/bin/sh` or `/bin/bash` are necessary. +### Simple debug console setup -### Enable agent debug console +Kata Containers 2.0 supports a shell simulated *console* for quick debug purpose. This approach uses VSOCK to +connect to the shell running inside the guest which the agent starts. This method only requires the guest image to +contain either `/bin/sh` or `/bin/bash`. -Change your `configuration.toml`, add agent debug parameters. +#### Enable agent debug console + +Enable debug_console_enabled in the configuration.toml configuration file: ``` -kernel_params = "agent.debug_console agent.debug_console_vport=1026" +[agent.kata] +debug_console_enabled = true ``` -Sandboxes created using this parameters will start a shell in guest if new connection is accept from `vsock`. +This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as kernel parameters, and sandboxes created using this parameters will start a shell in guest if new connection is accept from VSOCK. -### Start `kata-monitor` +#### Start `kata-monitor` -`kata-runitime exec` need `kata-monitor` to get the sandbox's `vsock` address to connect to, firt start `kata-monitor`. +The `kata-runtime exec` command needs `kata-monitor` to get the sandbox's `vsock` address to connect to, firt start `kata-monitor`. ``` $ sudo kata-monitor @@ -463,9 +470,9 @@ $ sudo kata-monitor `kata-monitor` will serve at `localhost:8090` by default. -### Connect to debug console +#### Connect to debug console -Command `kata-runitime exec` is used to connect to the debug console. +Command `kata-runtime exec` is used to connect to the debug console. ``` $ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd @@ -477,9 +484,9 @@ bash-4.2# exit exit ``` -If you want to access guest OS through a traditional way, see [Set up a debug console(the traditional way)](#set-up-a-debug-console-the-traditional-way). +If you want to access guest OS through a traditional way, see [Traditional debug console setup)](#traditional-debug-console-setup). -## Set up a debug console(the traditional way) +### Traditional debug console setup By default you cannot login to a virtual machine, since this can be sensitive from a security perspective. Also, allowing logins would require additional @@ -506,7 +513,7 @@ the following steps (using rootfs or initrd image). > > Once these steps are taken you can connect to the virtual machine using the [debug console](Developer-Guide.md#connect-to-the-virtual-machine-using-the-debug-console). -### Create a custom image containing a shell +#### Create a custom image containing a shell To login to a virtual machine, you must [create a custom rootfs](#create-a-rootfs-image) or [custom initrd](#create-an-initrd-image---optional) @@ -521,7 +528,7 @@ $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/too $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos' ``` -### Create a debug systemd service +#### Create a debug systemd service Create the service file that starts the shell in the rootfs directory: @@ -550,12 +557,12 @@ Add a dependency to start the debug console: $ sudo sed -i '$a Requires=kata-debug.service' ${ROOTFS_DIR}/lib/systemd/system/kata-containers.target ``` -### Build the debug image +#### Build the debug image Follow the instructions in the [Build a rootfs image](#build-a-rootfs-image) section when using rootfs, or when using initrd, complete the steps in the [Build an initrd image](#build-an-initrd-image) section. -### Configure runtime for custom debug image +#### Configure runtime for custom debug image Install the image: @@ -580,7 +587,7 @@ $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img) **Note**: You should take care to undo this change after you finish debugging to avoid all subsequently created containers from using the debug image. -### Create a container +#### Create a container Create a container as normal. For example using crictl: @@ -588,7 +595,7 @@ Create a container as normal. For example using crictl: $ sudo crictl run -r kata container.yaml pod.yaml ``` -### Connect to the virtual machine using the debug console +#### Connect to the virtual machine using the debug console ``` $ id=$(sudo crictl pods --no-trunc -q) @@ -601,7 +608,7 @@ $ sudo socat "stdin,raw,echo=0,escape=0x11" "unix-connect:${console}" To disconnect from the virtual machine, type `CONTROL+q` (hold down the `CONTROL` key and press `q`). -### Obtain details of the image +## Obtain details of the image If the image is created using [osbuilder](../tools/osbuilder), the following YAML diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 138527f247..946adefaa9 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -531,18 +531,21 @@ fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Re let master_fd = pseduo.master; let debug_shell_logger = logger.clone(); + // channel that used to sync between thread and main process + let (tx, rx) = mpsc::channel::(); + // start a thread to do IO copy between socket and pseduo.master thread::spawn(move || { let mut master_reader = unsafe { File::from_raw_fd(master_fd) }; - let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) }; - let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) }; let mut master_writer = unsafe { File::from_raw_fd(master_fd) }; + let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) }; + let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) }; loop { let mut fd_set = FdSet::new(); + fd_set.insert(rfd); fd_set.insert(master_fd); fd_set.insert(socket_fd); - fd_set.insert(rfd); match select( Some(fd_set.highest().unwrap() + 1), @@ -557,36 +560,7 @@ fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Re continue; } else { error!(debug_shell_logger, "select error {:?}", e); - break; - } - } - } - - if fd_set.contains(master_fd) { - match io_copy(&mut master_reader, &mut socket_writer) { - Ok(0) => { - debug!(debug_shell_logger, "master fd closed"); - break; - } - Ok(_) => {} - Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, - Err(e) => { - error!(debug_shell_logger, "read master fd error {:?}", e); - break; - } - } - } - - if fd_set.contains(socket_fd) { - match io_copy(&mut socket_reader, &mut master_writer) { - Ok(0) => { - debug!(debug_shell_logger, "master fd closed"); - break; - } - Ok(_) => {} - Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, - Err(e) => { - error!(debug_shell_logger, "read master fd error {:?}", e); + tx.send(1).unwrap(); break; } } @@ -595,22 +569,66 @@ fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Re if fd_set.contains(rfd) { info!( debug_shell_logger, - "debug shelll process {} exited", child_pid + "debug shell process {} exited", child_pid ); + tx.send(1).unwrap(); break; } + + if fd_set.contains(master_fd) { + match io_copy(&mut master_reader, &mut socket_writer) { + Ok(0) => { + debug!(debug_shell_logger, "master fd closed"); + tx.send(1).unwrap(); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read master fd error {:?}", e); + tx.send(1).unwrap(); + break; + } + } + } + + if fd_set.contains(socket_fd) { + match io_copy(&mut socket_reader, &mut master_writer) { + Ok(0) => { + debug!(debug_shell_logger, "socket fd closed"); + tx.send(1).unwrap(); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read socket fd error {:?}", e); + tx.send(1).unwrap(); + break; + } + } + } } }); let wait_status = wait::waitpid(child_pid, None); - info!(logger, "debug console exit code: {:?}", wait_status); + info!(logger, "debug console process exit code: {:?}", wait_status); + info!(logger, "notify debug monitor thread to exit"); // close pipe to exit select loop let _ = close(wfd); + + // wait for thread exit. + let _ = rx.recv().unwrap(); + info!(logger, "debug monitor thread has exited"); + + // close files + let _ = close(rfd); + let _ = close(master_fd); + let _ = close(slave_fd); } Err(err) => { - let msg = format!("fork error: {:?}", err); - return Err(ErrorKind::ErrorCode(msg).into()); + return Err(anyhow!("fork error: {:?}", err)); } } diff --git a/src/runtime/cli/config-generated.go.in b/src/runtime/cli/config-generated.go.in index fc69f13660..41ce38ac3b 100644 --- a/src/runtime/cli/config-generated.go.in +++ b/src/runtime/cli/config-generated.go.in @@ -39,7 +39,6 @@ var version = "@VERSION@" // project-specific command names var envCmd = fmt.Sprintf("%s-env", projectPrefix) var checkCmd = fmt.Sprintf("%s-check", projectPrefix) -var execCmd = "exec" // project-specific option names var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix) diff --git a/src/runtime/cli/config/configuration-acrn.toml.in b/src/runtime/cli/config/configuration-acrn.toml.in index 7e63021cb0..15fb00e45d 100644 --- a/src/runtime/cli/config/configuration-acrn.toml.in +++ b/src/runtime/cli/config/configuration-acrn.toml.in @@ -127,6 +127,13 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@" #trace_mode = "dynamic" #trace_type = "isolated" +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + [netmon] # If enabled, the network monitoring process gets started when the # sandbox is created. This allows for the detection of some additional diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 35518a67c5..d4d4e7978d 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -125,6 +125,12 @@ block_device_driver = "virtio-blk" #trace_mode = "dynamic" #trace_type = "isolated" +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true [netmon] # If enabled, the network monitoring process gets started when the diff --git a/src/runtime/cli/config/configuration-fc.toml.in b/src/runtime/cli/config/configuration-fc.toml.in index 068a05441e..e72130e0ed 100644 --- a/src/runtime/cli/config/configuration-fc.toml.in +++ b/src/runtime/cli/config/configuration-fc.toml.in @@ -256,6 +256,13 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_FC@" # kernel_modules=[] +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + [netmon] # If enabled, the network monitoring process gets started when the # sandbox is created. This allows for the detection of some additional diff --git a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in index c44b8b6044..a199bd30df 100644 --- a/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in +++ b/src/runtime/cli/config/configuration-qemu-virtiofs.toml.in @@ -352,6 +352,12 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # kernel_modules=[] +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true [netmon] # If enabled, the network monitoring process gets started when the diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 5e1fbf7027..0da8d50666 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -375,6 +375,12 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@" # kernel_modules=[] +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true [netmon] # If enabled, the network monitoring process gets started when the diff --git a/src/runtime/cli/kata-exec.go b/src/runtime/cli/kata-exec.go index ed3a7cabd5..44b0ab4b75 100644 --- a/src/runtime/cli/kata-exec.go +++ b/src/runtime/cli/kata-exec.go @@ -28,9 +28,16 @@ import ( const ( // The buffer size used to specify the buffer for IO streams copy - bufSize = 32 << 10 + bufSize = 1024 * 2 defaultTimeout = 3 * time.Second + + subCommandName = "exec" + // command-line parameters name + paramKataMonitorAddr = "kata-monitor-addr" + paramDebugConsolePort = "kata-debug-port" + defaultKernelParamDebugConsoleVPortValue = 1026 + defaultParamKataMonitorAddr = "http://localhost:8090" ) var ( @@ -43,15 +50,15 @@ var ( ) var kataExecCLICommand = cli.Command{ - Name: execCmd, + Name: subCommandName, Usage: "Enter into guest by debug console", Flags: []cli.Flag{ cli.StringFlag{ - Name: "monitor-addr", + Name: paramKataMonitorAddr, Usage: "Kata monitor listen address.", }, cli.Uint64Flag{ - Name: "debug-port", + Name: paramDebugConsolePort, Usage: "Port that debug console is listening on.", }, }, @@ -60,17 +67,17 @@ var kataExecCLICommand = cli.Command{ if err != nil { return err } - span, _ := katautils.Trace(ctx, "exec") + span, _ := katautils.Trace(ctx, subCommandName) defer span.Finish() - endPoint := context.String("monitor-addr") + endPoint := context.String(paramKataMonitorAddr) if endPoint == "" { - endPoint = "http://localhost:8090" + endPoint = defaultParamKataMonitorAddr } - port := context.Uint64("debug-port") + port := context.Uint64(paramDebugConsolePort) if port == 0 { - port = 1026 + port = defaultKernelParamDebugConsoleVPortValue } sandboxID := context.Args().Get(0) @@ -188,8 +195,11 @@ func getConn(endPoint, sandboxID string, port uint64) (net.Conn, error) { switch addr.Scheme { case clientUtils.VSockSocketScheme: // vsock://31513974:1024 - shimAddr := clientUtils.VSockSocketScheme + ":" + addr.Host - shimAddr = strings.Replace(shimAddr, ":1024", fmt.Sprintf(":%d", port), -1) + cidAndPort := strings.Split(addr.Host, ":") + if len(cidAndPort) != 2 { + return nil, fmt.Errorf("Invalid vsock scheme: %s", sock) + } + shimAddr := fmt.Sprintf("%s:%s:%d", clientUtils.VSockSocketScheme, cidAndPort[0], port) return clientUtils.VsockDialer(shimAddr, defaultTimeout) case clientUtils.HybridVSockScheme: diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e83940bd80..e967b081e9 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -132,11 +132,12 @@ type runtime struct { } type agent struct { - Debug bool `toml:"enable_debug"` - Tracing bool `toml:"enable_tracing"` - TraceMode string `toml:"trace_mode"` - TraceType string `toml:"trace_type"` - KernelModules []string `toml:"kernel_modules"` + Debug bool `toml:"enable_debug"` + Tracing bool `toml:"enable_tracing"` + TraceMode string `toml:"trace_mode"` + TraceType string `toml:"trace_type"` + KernelModules []string `toml:"kernel_modules"` + DebugConsoleEnabled bool `toml:"debug_console_enabled"` } type netmon struct { @@ -441,6 +442,10 @@ func (h hypervisor) getIOMMUPlatform() bool { return h.IOMMUPlatform } +func (a agent) debugConsoleEnabled() bool { + return a.DebugConsoleEnabled +} + func (a agent) debug() bool { return a.Debug } @@ -866,23 +871,15 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi } func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { - if builtIn { - config.AgentConfig = vc.KataAgentConfig{ - LongLiveConn: true, - Debug: config.AgentConfig.Debug, - KernelModules: config.AgentConfig.KernelModules, - } - - return nil - } - for _, agent := range tomlConf.Agent { config.AgentConfig = vc.KataAgentConfig{ - Debug: agent.debug(), - Trace: agent.trace(), - TraceMode: agent.traceMode(), - TraceType: agent.traceType(), - KernelModules: agent.kernelModules(), + LongLiveConn: true, + Debug: agent.debug(), + Trace: agent.trace(), + TraceMode: agent.traceMode(), + TraceType: agent.traceType(), + KernelModules: agent.kernelModules(), + EnableDebugConsole: agent.debugConsoleEnabled(), } } @@ -1026,12 +1023,10 @@ func initConfig() (config oci.RuntimeConfig, err error) { return oci.RuntimeConfig{}, err } - defaultAgentConfig := vc.KataAgentConfig{} - config = oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: GetDefaultHypervisorConfig(), - AgentConfig: defaultAgentConfig, + AgentConfig: vc.KataAgentConfig{}, } return config, nil diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 4a423aac9e..2fc36dec66 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -167,7 +167,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf VirtioFSCache: defaultVirtioFSCacheMode, } - agentConfig := vc.KataAgentConfig{} + agentConfig := vc.KataAgentConfig{ + LongLiveConn: true, + } netmonConfig := vc.NetmonConfig{ Path: netmonPath, @@ -519,7 +521,8 @@ func TestMinimalRuntimeConfig(t *testing.T) { # Runtime configuration file [agent.kata] - + debug_console_enabled=true + kernel_modules=["a", "b", "c"] [netmon] path = "` + netmonPath + `" ` @@ -576,7 +579,11 @@ func TestMinimalRuntimeConfig(t *testing.T) { VirtioFSCache: defaultVirtioFSCacheMode, } - expectedAgentConfig := vc.KataAgentConfig{} + expectedAgentConfig := vc.KataAgentConfig{ + LongLiveConn: true, + EnableDebugConsole: true, + KernelModules: []string{"a", "b", "c"}, + } expectedNetmonConfig := vc.NetmonConfig{ Path: netmonPath, diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 3e1fcd6202..b88bec4488 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -52,6 +52,11 @@ const ( // path to vfio devices vfioPath = "/dev/vfio/" + + // enable debug console + kernelParamDebugConsole = "agent.debug_console" + kernelParamDebugConsoleVPort = "agent.debug_console_vport" + kernelParamDebugConsoleVPortValue = "1026" ) var ( @@ -195,13 +200,14 @@ func ephemeralPath() string { // KataAgentConfig is a structure storing information needed // to reach the Kata Containers agent. type KataAgentConfig struct { - LongLiveConn bool - Debug bool - Trace bool - ContainerPipeSize uint32 - TraceMode string - TraceType string - KernelModules []string + LongLiveConn bool + Debug bool + Trace bool + EnableDebugConsole bool + ContainerPipeSize uint32 + TraceMode string + TraceType string + KernelModules []string } // KataAgentState is the structure describing the data stored from this @@ -294,6 +300,11 @@ func KataAgentKernelParams(config KataAgentConfig) []Param { params = append(params, Param{Key: vcAnnotations.ContainerPipeSizeKernelParam, Value: containerPipeSize}) } + if config.EnableDebugConsole { + params = append(params, Param{Key: kernelParamDebugConsole, Value: ""}) + params = append(params, Param{Key: kernelParamDebugConsoleVPort, Value: kernelParamDebugConsoleVPortValue}) + } + return params } @@ -1208,16 +1219,6 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat return nil, nil } -func (k *kataAgent) hasAgentDebugConsole(sandbox *Sandbox) bool { - for _, p := range sandbox.config.HypervisorConfig.KernelParams { - if p.Key == "agent.debug_console" { - k.Logger().Info("agent has debug console") - return true - } - } - return false -} - func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, err error) { span, _ := k.trace("createContainer") defer span.Finish() diff --git a/src/runtime/virtcontainers/vm_test.go b/src/runtime/virtcontainers/vm_test.go index a3fe6184ff..f592f2adce 100644 --- a/src/runtime/virtcontainers/vm_test.go +++ b/src/runtime/virtcontainers/vm_test.go @@ -101,7 +101,15 @@ func TestVMConfigGrpc(t *testing.T) { config := VMConfig{ HypervisorType: QemuHypervisor, HypervisorConfig: newQemuConfig(), - AgentConfig: KataAgentConfig{true, false, false, 0, "", "", []string{}}, + AgentConfig: KataAgentConfig{ + LongLiveConn: true, + Debug: false, + Trace: false, + EnableDebugConsole: false, + ContainerPipeSize: 0, + TraceMode: "", + TraceType: "", + KernelModules: []string{}}, } p, err := config.ToGrpc()