From 0b62f5a96fe1412c95bcfb95f304370d2ee86f26 Mon Sep 17 00:00:00 2001 From: bin liu Date: Thu, 16 Jul 2020 16:13:05 +0800 Subject: [PATCH] runtime: add debug console service Add `kata-runtime exec` to enter guest OS through shell started by agent Fixes: #245 Signed-off-by: bin liu --- docs/Developer-Guide.md | 51 ++++- src/agent/src/main.rs | 208 ++++++++++++++++-- src/runtime/cli/config-generated.go.in | 1 + src/runtime/cli/kata-exec.go | 208 ++++++++++++++++++ src/runtime/cli/kata-monitor/main.go | 1 + src/runtime/cli/main.go | 1 + .../containerd-shim-v2/shim_management.go | 14 ++ src/runtime/pkg/kata-monitor/metrics.go | 29 +-- src/runtime/pkg/kata-monitor/monitor.go | 22 ++ src/runtime/pkg/kata-monitor/pprof.go | 8 - src/runtime/pkg/kata-monitor/shim_client.go | 81 +++++++ src/runtime/virtcontainers/interfaces.go | 1 + .../pkg/agent/protocols/client/client.go | 4 +- .../pkg/annotations/annotations.go | 2 +- .../virtcontainers/pkg/vcmock/sandbox.go | 7 + .../virtcontainers/pkg/vcmock/types.go | 1 + src/runtime/virtcontainers/sandbox.go | 4 + 17 files changed, 578 insertions(+), 65 deletions(-) create mode 100644 src/runtime/cli/kata-exec.go create mode 100644 src/runtime/pkg/kata-monitor/shim_client.go diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 7ff1ef9463..c5cb3583e7 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -34,7 +34,11 @@ * [Troubleshoot Kata Containers](#troubleshoot-kata-containers) * [Appendices](#appendices) * [Checking Docker default runtime](#checking-docker-default-runtime) - * [Set up a debug console](#set-up-a-debug-console) + * [Set up a debug console(the easy way)](#set-up-a-debug-consolethe-easy-way) + * [Enable agent debug console](#enable-agent-debug-console) + * [Start `kata-monitor`](#start-kata-monitor) + * [Connect to debug console](#connect-to-debug-console) + * [Set up a debug console(the traditional way)](#set-up-a-debug-consolethe-traditional-way) * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) * [Create a debug systemd service](#create-a-debug-systemd-service) * [Build the debug image](#build-the-debug-image) @@ -60,7 +64,7 @@ The recommended way to create a development environment is to first to create a working system. The installation guide instructions will install all required Kata Containers -components, plus Docker*, the hypervisor, and the Kata Containers image and +components, plus *Docker*, the hypervisor, and the Kata Containers image and guest kernel. # Requirements to build individual components @@ -434,7 +438,48 @@ See [Set up a debug console](#set-up-a-debug-console). $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime" ``` -## Set up a debug console +## Set up a debug console(The easy way) + +Kata containers 2.0 support a shell simulated *console* for quickly debug purpose. This approach use `vsock` to connect shell running inside guest started by agent. The good aspect is that we need not modify guest image or despite using what device that hypervisors support. Only `/bin/sh` or `/bin/bash` are necessary. + +### Enable agent debug console + +Change your `configuration.toml`, add agent debug parameters. + +``` +kernel_params = "agent.debug_console agent.debug_console_vport=1026" +``` + +Sandboxes created using this parameters will start a shell in guest if new connection is accept from `vsock`. + +### Start `kata-monitor` + +`kata-runitime exec` need `kata-monitor` to get the sandbox's `vsock` address to connect to, firt start `kata-monitor`. + +``` +$ sudo kata-monitor +``` + +`kata-monitor` will serve at `localhost:8090` by default. + + +### Connect to debug console + +Command `kata-runitime exec` is used to connect to the debug console. + +``` +$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd +bash-4.2# id +uid=0(root) gid=0(root) groups=0(root) +bash-4.2# pwd +/ +bash-4.2# exit +exit +``` + +If you want to access guest OS through a traditional way, see [Set up a debug console(the traditional way)](#set-up-a-debug-console-the-traditional-way). + +## Set up a debug console(the traditional way) By default you cannot login to a virtual machine, since this can be sensitive from a security perspective. Also, allowing logins would require additional diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 25b6218b51..138527f247 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -31,16 +31,20 @@ extern crate netlink; use crate::netlink::{RtnlHandle, NETLINK_ROUTE}; use anyhow::{anyhow, Context, Result}; use nix::fcntl::{self, OFlag}; +use nix::fcntl::{FcntlArg, FdFlag}; +use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use nix::pty; +use nix::sys::select::{select, FdSet}; use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType}; use nix::sys::wait::{self, WaitStatus}; -use nix::unistd; -use nix::unistd::dup; +use nix::unistd::{self, close, dup, dup2, fork, setsid, ForkResult}; use prctl::set_child_subreaper; use signal_hook::{iterator::Signals, SIGCHLD}; use std::collections::HashMap; use std::env; -use std::ffi::OsStr; +use std::ffi::{CStr, CString, OsStr}; use std::fs::{self, File}; +use std::io::{Read, Write}; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs as unixfs; use std::os::unix::io::AsRawFd; @@ -75,6 +79,8 @@ const NAME: &str = "kata-agent"; const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline"; const CONSOLE_PATH: &str = "/dev/console"; +const DEFAULT_BUF_SIZE: usize = 8 * 1024; + lazy_static! { static ref GLOBAL_DEVICE_WATCHER: Arc>>> = Arc::new(Mutex::new(HashMap::new())); @@ -213,7 +219,7 @@ fn start_sandbox(logger: &Logger, config: &agentConfig, init_mode: bool) -> Resu let handle = builder.spawn(move || { let shells = shells.lock().unwrap(); - let result = setup_debug_console(shells.to_vec(), debug_console_vport); + let result = setup_debug_console(&thread_logger, shells.to_vec(), debug_console_vport); if result.is_err() { // Report error, but don't fail warn!(thread_logger, "failed to setup debug console"; @@ -406,9 +412,9 @@ use crate::config::agentConfig; use nix::sys::stat::Mode; use std::os::unix::io::{FromRawFd, RawFd}; use std::path::PathBuf; -use std::process::{exit, Command, Stdio}; +use std::process::exit; -fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { +fn setup_debug_console(logger: &Logger, shells: Vec, port: u32) -> Result<()> { let mut shell: &str = ""; for sh in shells.iter() { let binary = PathBuf::from(sh); @@ -422,7 +428,7 @@ fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { return Err(anyhow!("no shell found to launch debug console")); } - let f: RawFd = if port > 0 { + if port > 0 { let listenfd = socket::socket( AddressFamily::Vsock, SockType::Stream, @@ -432,29 +438,183 @@ fn setup_debug_console(shells: Vec, port: u32) -> Result<()> { let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port); socket::bind(listenfd, &addr)?; socket::listen(listenfd, 1)?; - socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)? + loop { + let f: RawFd = socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?; + match run_debug_console_shell(logger, shell, f) { + Ok(_) => { + info!(logger, "run_debug_console_shell session finished"); + } + Err(err) => { + error!(logger, "run_debug_console_shell failed: {:?}", err); + } + } + } } else { let mut flags = OFlag::empty(); flags.insert(OFlag::O_RDWR); flags.insert(OFlag::O_CLOEXEC); - fcntl::open(CONSOLE_PATH, flags, Mode::empty())? + loop { + let f: RawFd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?; + match run_debug_console_shell(logger, shell, f) { + Ok(_) => { + info!(logger, "run_debug_console_shell session finished"); + } + Err(err) => { + error!(logger, "run_debug_console_shell failed: {:?}", err); + } + } + } + }; +} + +fn io_copy(reader: &mut R, writer: &mut W) -> io::Result +where + R: Read, + W: Write, +{ + let mut buf = [0; DEFAULT_BUF_SIZE]; + let buf_len; + + match reader.read(&mut buf) { + Ok(0) => return Ok(0), + Ok(len) => buf_len = len, + Err(err) => return Err(err), }; - let cmd = Command::new(shell) - .arg("-i") - .stdin(unsafe { Stdio::from_raw_fd(f) }) - .stdout(unsafe { Stdio::from_raw_fd(f) }) - .stderr(unsafe { Stdio::from_raw_fd(f) }) - .spawn(); + // write and return + match writer.write_all(&buf[..buf_len]) { + Ok(_) => return Ok(buf_len as u64), + Err(err) => return Err(err), + } +} - let mut cmd = match cmd { - Ok(c) => c, - Err(_) => return Err(anyhow!("failed to spawn shell")), - }; +fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Result<()> { + let pseduo = pty::openpty(None, None)?; + let _ = fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); + let _ = fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC)); - cmd.wait()?; + let slave_fd = pseduo.slave; - return Ok(()); + match fork() { + Ok(ForkResult::Child) => { + // create new session with child as session leader + setsid()?; + + // dup stdin, stdout, stderr to let child act as a terminal + dup2(slave_fd, STDIN_FILENO)?; + dup2(slave_fd, STDOUT_FILENO)?; + dup2(slave_fd, STDERR_FILENO)?; + + // set tty + unsafe { + libc::ioctl(0, libc::TIOCSCTTY); + } + + let cmd = CString::new(shell).unwrap(); + let args: Vec<&CStr> = vec![]; + + // run shell + if let Err(e) = unistd::execvp(cmd.as_c_str(), args.as_slice()) { + match e { + nix::Error::Sys(errno) => { + std::process::exit(errno as i32); + } + _ => std::process::exit(-2), + } + } + } + + Ok(ForkResult::Parent { child: child_pid }) => { + info!(logger, "get debug shell pid {:?}", child_pid); + + let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?; + let master_fd = pseduo.master; + let debug_shell_logger = logger.clone(); + + // start a thread to do IO copy between socket and pseduo.master + thread::spawn(move || { + let mut master_reader = unsafe { File::from_raw_fd(master_fd) }; + let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) }; + let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) }; + let mut master_writer = unsafe { File::from_raw_fd(master_fd) }; + + loop { + let mut fd_set = FdSet::new(); + fd_set.insert(master_fd); + fd_set.insert(socket_fd); + fd_set.insert(rfd); + + match select( + Some(fd_set.highest().unwrap() + 1), + &mut fd_set, + None, + None, + None, + ) { + Ok(_) => (), + Err(e) => { + if e == nix::Error::from(nix::errno::Errno::EINTR) { + continue; + } else { + error!(debug_shell_logger, "select error {:?}", e); + break; + } + } + } + + if fd_set.contains(master_fd) { + match io_copy(&mut master_reader, &mut socket_writer) { + Ok(0) => { + debug!(debug_shell_logger, "master fd closed"); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read master fd error {:?}", e); + break; + } + } + } + + if fd_set.contains(socket_fd) { + match io_copy(&mut socket_reader, &mut master_writer) { + Ok(0) => { + debug!(debug_shell_logger, "master fd closed"); + break; + } + Ok(_) => {} + Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + error!(debug_shell_logger, "read master fd error {:?}", e); + break; + } + } + } + + if fd_set.contains(rfd) { + info!( + debug_shell_logger, + "debug shelll process {} exited", child_pid + ); + break; + } + } + }); + + let wait_status = wait::waitpid(child_pid, None); + info!(logger, "debug console exit code: {:?}", wait_status); + + // close pipe to exit select loop + let _ = close(wfd); + } + Err(err) => { + let msg = format!("fork error: {:?}", err); + return Err(ErrorKind::ErrorCode(msg).into()); + } + } + + Ok(()) } #[cfg(test)] @@ -472,8 +632,9 @@ mod tests { let shells_ref = SHELLS.clone(); let mut shells = shells_ref.lock().unwrap(); shells.clear(); + let logger = slog_scope::logger(); - let result = setup_debug_console(shells.to_vec(), 0); + let result = setup_debug_console(&logger, shells.to_vec(), 0); assert!(result.is_err()); assert_eq!( @@ -498,8 +659,9 @@ mod tests { .to_string(); shells.push(shell); + let logger = slog_scope::logger(); - let result = setup_debug_console(shells.to_vec(), 0); + let result = setup_debug_console(&logger, shells.to_vec(), 0); assert!(result.is_err()); assert_eq!( diff --git a/src/runtime/cli/config-generated.go.in b/src/runtime/cli/config-generated.go.in index 41ce38ac3b..fc69f13660 100644 --- a/src/runtime/cli/config-generated.go.in +++ b/src/runtime/cli/config-generated.go.in @@ -39,6 +39,7 @@ var version = "@VERSION@" // project-specific command names var envCmd = fmt.Sprintf("%s-env", projectPrefix) var checkCmd = fmt.Sprintf("%s-check", projectPrefix) +var execCmd = "exec" // project-specific option names var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix) diff --git a/src/runtime/cli/kata-exec.go b/src/runtime/cli/kata-exec.go new file mode 100644 index 0000000000..ed3a7cabd5 --- /dev/null +++ b/src/runtime/cli/kata-exec.go @@ -0,0 +1,208 @@ +// Copyright (c) 2017-2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strings" + + "sync" + "time" + + "github.com/containerd/console" + "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +const ( + + // The buffer size used to specify the buffer for IO streams copy + bufSize = 32 << 10 + + defaultTimeout = 3 * time.Second +) + +var ( + bufPool = sync.Pool{ + New: func() interface{} { + buffer := make([]byte, bufSize) + return &buffer + }, + } +) + +var kataExecCLICommand = cli.Command{ + Name: execCmd, + Usage: "Enter into guest by debug console", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "monitor-addr", + Usage: "Kata monitor listen address.", + }, + cli.Uint64Flag{ + Name: "debug-port", + Usage: "Port that debug console is listening on.", + }, + }, + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { + return err + } + span, _ := katautils.Trace(ctx, "exec") + defer span.Finish() + + endPoint := context.String("monitor-addr") + if endPoint == "" { + endPoint = "http://localhost:8090" + } + + port := context.Uint64("debug-port") + if port == 0 { + port = 1026 + } + + sandboxID := context.Args().Get(0) + if sandboxID == "" { + return fmt.Errorf("SandboxID not found") + } + + conn, err := getConn(endPoint, sandboxID, port) + if err != nil { + return err + } + defer conn.Close() + + con := console.Current() + defer con.Reset() + + if err := con.SetRaw(); err != nil { + return err + } + + iostream := &iostream{ + conn: conn, + exitch: make(chan struct{}), + closed: false, + } + + ioCopy(iostream, con) + + <-iostream.exitch + return nil + }, +} + +func ioCopy(stream *iostream, con console.Console) { + var wg sync.WaitGroup + + // stdin + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + io.CopyBuffer(stream, con, *p) + }() + + // stdout + wg.Add(1) + go func() { + p := bufPool.Get().(*[]byte) + defer bufPool.Put(p) + io.CopyBuffer(os.Stdout, stream, *p) + wg.Done() + }() + + wg.Wait() + close(stream.exitch) +} + +type iostream struct { + conn net.Conn + exitch chan struct{} + closed bool +} + +func (s *iostream) Write(data []byte) (n int, err error) { + if s.closed { + return 0, errors.New("stream closed") + } + return s.conn.Write(data) +} + +func (s *iostream) Close() error { + if s.closed { + return errors.New("stream closed") + } + + err := s.conn.Close() + if err == nil { + s.closed = true + } + + return err +} + +func (s *iostream) Read(data []byte) (n int, err error) { + if s.closed { + return 0, errors.New("stream closed") + } + + return s.conn.Read(data) +} + +func getConn(endPoint, sandboxID string, port uint64) (net.Conn, error) { + shimURL := fmt.Sprintf("%s/agent-url?sandbox=%s", endPoint, sandboxID) + resp, err := http.Get(shimURL) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Failed to get %s: %d", shimURL, resp.StatusCode) + } + + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + sock := strings.TrimSuffix(string(data), "\n") + addr, err := url.Parse(sock) + if err != nil { + return nil, err + } + + // validate more + switch addr.Scheme { + case clientUtils.VSockSocketScheme: + // vsock://31513974:1024 + shimAddr := clientUtils.VSockSocketScheme + ":" + addr.Host + shimAddr = strings.Replace(shimAddr, ":1024", fmt.Sprintf(":%d", port), -1) + return clientUtils.VsockDialer(shimAddr, defaultTimeout) + + case clientUtils.HybridVSockScheme: + // addr: hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock:1024 + hvsocket := strings.Split(addr.Path, ":") + if len(hvsocket) != 2 { + return nil, fmt.Errorf("Invalid hybrid vsock scheme: %s", sock) + } + + // hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock + shimAddr := fmt.Sprintf("%s:%s:%d", clientUtils.HybridVSockScheme, hvsocket[0], port) + return clientUtils.HybridVSockDialer(shimAddr, defaultTimeout) + } + + return nil, fmt.Errorf("schema %s not found", addr.Scheme) +} diff --git a/src/runtime/cli/kata-monitor/main.go b/src/runtime/cli/kata-monitor/main.go index 1b5920361e..6f66504b03 100644 --- a/src/runtime/cli/kata-monitor/main.go +++ b/src/runtime/cli/kata-monitor/main.go @@ -36,6 +36,7 @@ func main() { m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(km.ProcessMetricsRequest)) m.Handle("/sandboxes", http.HandlerFunc(km.ListSandboxes)) + m.Handle("/agent-url", http.HandlerFunc(km.GetAgentURL)) // for debug shim process m.Handle("/debug/vars", http.HandlerFunc(km.ExpvarHandler)) diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go index a8b3c42c2a..2b721eaa21 100644 --- a/src/runtime/cli/main.go +++ b/src/runtime/cli/main.go @@ -125,6 +125,7 @@ var runtimeCommands = []cli.Command{ // Kata Containers specific extensions kataCheckCLICommand, kataEnvCLICommand, + kataExecCLICommand, factoryCLICommand, } diff --git a/src/runtime/containerd-shim-v2/shim_management.go b/src/runtime/containerd-shim-v2/shim_management.go index 9afaf3b4b9..f971a2a602 100644 --- a/src/runtime/containerd-shim-v2/shim_management.go +++ b/src/runtime/containerd-shim-v2/shim_management.go @@ -8,6 +8,7 @@ package containerdshim import ( "context" "expvar" + "fmt" "io" "net/http" "net/http/pprof" @@ -34,6 +35,18 @@ var ( shimMgtLog = shimLog.WithField("subsystem", "shim-management") ) +// agentURL returns URL for agent +func (s *service) agentURL(w http.ResponseWriter, r *http.Request) { + url, err := s.sandbox.GetAgentURL() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + fmt.Fprint(w, url) +} + // serveMetrics handle /metrics requests func (s *service) serveMetrics(w http.ResponseWriter, r *http.Request) { @@ -139,6 +152,7 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec // bind hanlder m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) + m.Handle("/agent-url", http.HandlerFunc(s.agentURL)) s.mountPprofHandle(m, ociSpec) // register shim metrics diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index 712924ffed..b4fcd5830a 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -10,7 +10,6 @@ import ( "compress/gzip" "io" "io/ioutil" - "net" "net/http" "path/filepath" "sort" @@ -236,33 +235,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { // getSandboxMetrics will get sandbox's metrics from shim func (km *KataMonitor) getSandboxMetrics(sandboxID, namespace string) ([]*dto.MetricFamily, error) { - socket, err := km.getMonitorAddress(sandboxID, namespace) - if err != nil { - return nil, err - } - - transport := &http.Transport{ - DisableKeepAlives: true, - Dial: func(proto, addr string) (conn net.Conn, err error) { - return net.Dial("unix", "\x00"+socket) - }, - } - - client := http.Client{ - Timeout: 3 * time.Second, - Transport: transport, - } - - resp, err := client.Get("http://shim/metrics") - if err != nil { - return nil, err - } - - defer func() { - resp.Body.Close() - }() - - body, err := ioutil.ReadAll(resp.Body) + body, err := km.doGet(sandboxID, namespace, defaultTimeout, "metrics") if err != nil { return nil, err } diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index 7f4cb4eeaf..3254b52021 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -80,6 +80,28 @@ func (km *KataMonitor) initSandboxCache() error { return nil } +// GetAgentURL returns agent URL +func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) { + sandboxID, err := getSandboxIdFromReq(r) + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + namespace, err := km.getSandboxNamespace(sandboxID) + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + + data, err := km.doGet(sandboxID, namespace, defaultTimeout, "agent-url") + if err != nil { + commonServeError(w, http.StatusBadRequest, err) + return + } + + fmt.Fprintln(w, string(data)) +} + // ListSandboxes list all sandboxes running in Kata func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) { sandboxes := km.getSandboxList() diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go index e043251cb7..9e54315a42 100644 --- a/src/runtime/pkg/kata-monitor/pprof.go +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -12,14 +12,6 @@ import ( "net/http" ) -func getSandboxIdFromReq(r *http.Request) (string, error) { - sandbox := r.URL.Query().Get("sandbox") - if sandbox != "" { - return sandbox, nil - } - return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query()) -} - func serveError(w http.ResponseWriter, status int, txt string) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("X-Go-Pprof", "1") diff --git a/src/runtime/pkg/kata-monitor/shim_client.go b/src/runtime/pkg/kata-monitor/shim_client.go new file mode 100644 index 0000000000..0c1c1c81a7 --- /dev/null +++ b/src/runtime/pkg/kata-monitor/shim_client.go @@ -0,0 +1,81 @@ +// Copyright (c) 2020 Ant Financial +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "time" +) + +const ( + defaultTimeout = 3 * time.Second +) + +func commonServeError(w http.ResponseWriter, status int, err error) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(status) + if err != nil { + fmt.Fprintln(w, err.Error()) + } +} + +func getSandboxIdFromReq(r *http.Request) (string, error) { + sandbox := r.URL.Query().Get("sandbox") + if sandbox != "" { + return sandbox, nil + } + return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query()) +} + +func (km *KataMonitor) buildShimClient(sandboxID, namespace string, timeout time.Duration) (*http.Client, error) { + socket, err := km.getMonitorAddress(sandboxID, namespace) + if err != nil { + return nil, err + } + + transport := &http.Transport{ + DisableKeepAlives: true, + Dial: func(proto, addr string) (conn net.Conn, err error) { + return net.Dial("unix", "\x00"+socket) + }, + } + + client := &http.Client{ + Transport: transport, + } + + if timeout > 0 { + client.Timeout = timeout + } + + return client, nil +} + +func (km *KataMonitor) doGet(sandboxID, namespace string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { + client, err := km.buildShimClient(sandboxID, namespace, timeoutInSeconds) + if err != nil { + return nil, err + } + + resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) + if err != nil { + return nil, err + } + + defer func() { + resp.Body.Close() + }() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/src/runtime/virtcontainers/interfaces.go b/src/runtime/virtcontainers/interfaces.go index c04acb676e..365c329db0 100644 --- a/src/runtime/virtcontainers/interfaces.go +++ b/src/runtime/virtcontainers/interfaces.go @@ -75,6 +75,7 @@ type VCSandbox interface { UpdateRuntimeMetrics() error GetAgentMetrics() (string, error) + GetAgentURL() (string, error) } // VCContainer is the Container interface diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go index 200f4584b6..c3b1c9390b 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/client/client.go @@ -178,7 +178,7 @@ func parse(sock string) (string, *url.URL, error) { func agentDialer(addr *url.URL) dialer { switch addr.Scheme { case VSockSocketScheme: - return vsockDialer + return VsockDialer case HybridVSockScheme: return HybridVSockDialer case MockHybridVSockScheme: @@ -278,7 +278,7 @@ func commonDialer(timeout time.Duration, dialFunc func() (net.Conn, error), time return conn, nil } -func vsockDialer(sock string, timeout time.Duration) (net.Conn, error) { +func VsockDialer(sock string, timeout time.Duration) (net.Conn, error) { cid, port, err := parseGrpcVsockAddr(sock) if err != nil { return nil, err diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index c84d9d594e..b9b3bf2fa3 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -245,7 +245,7 @@ const ( // The following example can be used to load two kernel modules with parameters /// // annotations: - // io.kata-containers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" + // io.katacontainers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0" // // The first word is considered as the module name and the rest as its parameters. // diff --git a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go index a7d4c6b7e4..727fed7af8 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go +++ b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go @@ -247,3 +247,10 @@ func (s *Sandbox) Stats() (vc.SandboxStats, error) { } return vc.SandboxStats{}, nil } + +func (s *Sandbox) GetAgentURL() (string, error) { + if s.GetAgentURLFunc != nil { + return s.GetAgentURLFunc() + } + return "", nil +} diff --git a/src/runtime/virtcontainers/pkg/vcmock/types.go b/src/runtime/virtcontainers/pkg/vcmock/types.go index 39ea6fc4f5..211d97c8e2 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/types.go +++ b/src/runtime/virtcontainers/pkg/vcmock/types.go @@ -67,6 +67,7 @@ type Sandbox struct { UpdateRuntimeMetricsFunc func() error GetAgentMetricsFunc func() (string, error) StatsFunc func() (vc.SandboxStats, error) + GetAgentURLFunc func() (string, error) } // Container is a fake Container type used for testing diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a2582a105f..a393e5c76b 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2271,3 +2271,7 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec { func (s *Sandbox) GetOOMEvent() (string, error) { return s.agent.getOOMEvent() } + +func (s *Sandbox) GetAgentURL() (string, error) { + return s.agent.getAgentURL() +}