mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-06-19 12:14:11 +00:00
runtime: add debug console service
Add `kata-runtime exec` to enter guest OS through shell started by agent Fixes: #245 Signed-off-by: bin liu <bin@hyper.sh>
This commit is contained in:
parent
594519d883
commit
febdf8f68c
@ -34,7 +34,11 @@
|
||||
* [Troubleshoot Kata Containers](#troubleshoot-kata-containers)
|
||||
* [Appendices](#appendices)
|
||||
* [Checking Docker default runtime](#checking-docker-default-runtime)
|
||||
* [Set up a debug console](#set-up-a-debug-console)
|
||||
* [Set up a debug console(the easy way)](#set-up-a-debug-consolethe-easy-way)
|
||||
* [Enable agent debug console](#enable-agent-debug-console)
|
||||
* [Start `kata-monitor`](#start-kata-monitor)
|
||||
* [Connect to debug console](#connect-to-debug-console)
|
||||
* [Set up a debug console(the traditional way)](#set-up-a-debug-consolethe-traditional-way)
|
||||
* [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell)
|
||||
* [Create a debug systemd service](#create-a-debug-systemd-service)
|
||||
* [Build the debug image](#build-the-debug-image)
|
||||
@ -60,7 +64,7 @@ The recommended way to create a development environment is to first
|
||||
to create a working system.
|
||||
|
||||
The installation guide instructions will install all required Kata Containers
|
||||
components, plus Docker*, the hypervisor, and the Kata Containers image and
|
||||
components, plus *Docker*, the hypervisor, and the Kata Containers image and
|
||||
guest kernel.
|
||||
|
||||
# Requirements to build individual components
|
||||
@ -434,7 +438,48 @@ See [Set up a debug console](#set-up-a-debug-console).
|
||||
$ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
|
||||
```
|
||||
|
||||
## Set up a debug console
|
||||
## Set up a debug console(The easy way)
|
||||
|
||||
Kata containers 2.0 support a shell simulated *console* for quickly debug purpose. This approach use `vsock` to connect shell running inside guest started by agent. The good aspect is that we need not modify guest image or despite using what device that hypervisors support. Only `/bin/sh` or `/bin/bash` are necessary.
|
||||
|
||||
### Enable agent debug console
|
||||
|
||||
Change your `configuration.toml`, add agent debug parameters.
|
||||
|
||||
```
|
||||
kernel_params = "agent.debug_console agent.debug_console_vport=1026"
|
||||
```
|
||||
|
||||
Sandboxes created using this parameters will start a shell in guest if new connection is accept from `vsock`.
|
||||
|
||||
### Start `kata-monitor`
|
||||
|
||||
`kata-runitime exec` need `kata-monitor` to get the sandbox's `vsock` address to connect to, firt start `kata-monitor`.
|
||||
|
||||
```
|
||||
$ sudo kata-monitor
|
||||
```
|
||||
|
||||
`kata-monitor` will serve at `localhost:8090` by default.
|
||||
|
||||
|
||||
### Connect to debug console
|
||||
|
||||
Command `kata-runitime exec` is used to connect to the debug console.
|
||||
|
||||
```
|
||||
$ kata-runtime exec 1a9ab65be63b8b03dfd0c75036d27f0ed09eab38abb45337fea83acd3cd7bacd
|
||||
bash-4.2# id
|
||||
uid=0(root) gid=0(root) groups=0(root)
|
||||
bash-4.2# pwd
|
||||
/
|
||||
bash-4.2# exit
|
||||
exit
|
||||
```
|
||||
|
||||
If you want to access guest OS through a traditional way, see [Set up a debug console(the traditional way)](#set-up-a-debug-console-the-traditional-way).
|
||||
|
||||
## Set up a debug console(the traditional way)
|
||||
|
||||
By default you cannot login to a virtual machine, since this can be sensitive
|
||||
from a security perspective. Also, allowing logins would require additional
|
||||
|
@ -31,16 +31,20 @@ extern crate netlink;
|
||||
use crate::netlink::{RtnlHandle, NETLINK_ROUTE};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use nix::fcntl::{self, OFlag};
|
||||
use nix::fcntl::{FcntlArg, FdFlag};
|
||||
use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
|
||||
use nix::pty;
|
||||
use nix::sys::select::{select, FdSet};
|
||||
use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
|
||||
use nix::sys::wait::{self, WaitStatus};
|
||||
use nix::unistd;
|
||||
use nix::unistd::dup;
|
||||
use nix::unistd::{self, close, dup, dup2, fork, setsid, ForkResult};
|
||||
use prctl::set_child_subreaper;
|
||||
use signal_hook::{iterator::Signals, SIGCHLD};
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::ffi::OsStr;
|
||||
use std::ffi::{CStr, CString, OsStr};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::fs as unixfs;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
@ -75,6 +79,8 @@ const NAME: &str = "kata-agent";
|
||||
const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline";
|
||||
const CONSOLE_PATH: &str = "/dev/console";
|
||||
|
||||
const DEFAULT_BUF_SIZE: usize = 8 * 1024;
|
||||
|
||||
lazy_static! {
|
||||
static ref GLOBAL_DEVICE_WATCHER: Arc<Mutex<HashMap<String, Sender<String>>>> =
|
||||
Arc::new(Mutex::new(HashMap::new()));
|
||||
@ -213,7 +219,7 @@ fn start_sandbox(logger: &Logger, config: &agentConfig, init_mode: bool) -> Resu
|
||||
|
||||
let handle = builder.spawn(move || {
|
||||
let shells = shells.lock().unwrap();
|
||||
let result = setup_debug_console(shells.to_vec(), debug_console_vport);
|
||||
let result = setup_debug_console(&thread_logger, shells.to_vec(), debug_console_vport);
|
||||
if result.is_err() {
|
||||
// Report error, but don't fail
|
||||
warn!(thread_logger, "failed to setup debug console";
|
||||
@ -406,9 +412,9 @@ use crate::config::agentConfig;
|
||||
use nix::sys::stat::Mode;
|
||||
use std::os::unix::io::{FromRawFd, RawFd};
|
||||
use std::path::PathBuf;
|
||||
use std::process::{exit, Command, Stdio};
|
||||
use std::process::exit;
|
||||
|
||||
fn setup_debug_console(shells: Vec<String>, port: u32) -> Result<()> {
|
||||
fn setup_debug_console(logger: &Logger, shells: Vec<String>, port: u32) -> Result<()> {
|
||||
let mut shell: &str = "";
|
||||
for sh in shells.iter() {
|
||||
let binary = PathBuf::from(sh);
|
||||
@ -422,7 +428,7 @@ fn setup_debug_console(shells: Vec<String>, port: u32) -> Result<()> {
|
||||
return Err(anyhow!("no shell found to launch debug console"));
|
||||
}
|
||||
|
||||
let f: RawFd = if port > 0 {
|
||||
if port > 0 {
|
||||
let listenfd = socket::socket(
|
||||
AddressFamily::Vsock,
|
||||
SockType::Stream,
|
||||
@ -432,29 +438,183 @@ fn setup_debug_console(shells: Vec<String>, port: u32) -> Result<()> {
|
||||
let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
|
||||
socket::bind(listenfd, &addr)?;
|
||||
socket::listen(listenfd, 1)?;
|
||||
socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?
|
||||
loop {
|
||||
let f: RawFd = socket::accept4(listenfd, SockFlag::SOCK_CLOEXEC)?;
|
||||
match run_debug_console_shell(logger, shell, f) {
|
||||
Ok(_) => {
|
||||
info!(logger, "run_debug_console_shell session finished");
|
||||
}
|
||||
Err(err) => {
|
||||
error!(logger, "run_debug_console_shell failed: {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let mut flags = OFlag::empty();
|
||||
flags.insert(OFlag::O_RDWR);
|
||||
flags.insert(OFlag::O_CLOEXEC);
|
||||
fcntl::open(CONSOLE_PATH, flags, Mode::empty())?
|
||||
loop {
|
||||
let f: RawFd = fcntl::open(CONSOLE_PATH, flags, Mode::empty())?;
|
||||
match run_debug_console_shell(logger, shell, f) {
|
||||
Ok(_) => {
|
||||
info!(logger, "run_debug_console_shell session finished");
|
||||
}
|
||||
Err(err) => {
|
||||
error!(logger, "run_debug_console_shell failed: {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn io_copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> io::Result<u64>
|
||||
where
|
||||
R: Read,
|
||||
W: Write,
|
||||
{
|
||||
let mut buf = [0; DEFAULT_BUF_SIZE];
|
||||
let buf_len;
|
||||
|
||||
match reader.read(&mut buf) {
|
||||
Ok(0) => return Ok(0),
|
||||
Ok(len) => buf_len = len,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let cmd = Command::new(shell)
|
||||
.arg("-i")
|
||||
.stdin(unsafe { Stdio::from_raw_fd(f) })
|
||||
.stdout(unsafe { Stdio::from_raw_fd(f) })
|
||||
.stderr(unsafe { Stdio::from_raw_fd(f) })
|
||||
.spawn();
|
||||
// write and return
|
||||
match writer.write_all(&buf[..buf_len]) {
|
||||
Ok(_) => return Ok(buf_len as u64),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
let mut cmd = match cmd {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Err(anyhow!("failed to spawn shell")),
|
||||
};
|
||||
fn run_debug_console_shell(logger: &Logger, shell: &str, socket_fd: RawFd) -> Result<()> {
|
||||
let pseduo = pty::openpty(None, None)?;
|
||||
let _ = fcntl::fcntl(pseduo.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
|
||||
let _ = fcntl::fcntl(pseduo.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC));
|
||||
|
||||
cmd.wait()?;
|
||||
let slave_fd = pseduo.slave;
|
||||
|
||||
return Ok(());
|
||||
match fork() {
|
||||
Ok(ForkResult::Child) => {
|
||||
// create new session with child as session leader
|
||||
setsid()?;
|
||||
|
||||
// dup stdin, stdout, stderr to let child act as a terminal
|
||||
dup2(slave_fd, STDIN_FILENO)?;
|
||||
dup2(slave_fd, STDOUT_FILENO)?;
|
||||
dup2(slave_fd, STDERR_FILENO)?;
|
||||
|
||||
// set tty
|
||||
unsafe {
|
||||
libc::ioctl(0, libc::TIOCSCTTY);
|
||||
}
|
||||
|
||||
let cmd = CString::new(shell).unwrap();
|
||||
let args: Vec<&CStr> = vec![];
|
||||
|
||||
// run shell
|
||||
if let Err(e) = unistd::execvp(cmd.as_c_str(), args.as_slice()) {
|
||||
match e {
|
||||
nix::Error::Sys(errno) => {
|
||||
std::process::exit(errno as i32);
|
||||
}
|
||||
_ => std::process::exit(-2),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ForkResult::Parent { child: child_pid }) => {
|
||||
info!(logger, "get debug shell pid {:?}", child_pid);
|
||||
|
||||
let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC)?;
|
||||
let master_fd = pseduo.master;
|
||||
let debug_shell_logger = logger.clone();
|
||||
|
||||
// start a thread to do IO copy between socket and pseduo.master
|
||||
thread::spawn(move || {
|
||||
let mut master_reader = unsafe { File::from_raw_fd(master_fd) };
|
||||
let mut socket_writer = unsafe { File::from_raw_fd(socket_fd) };
|
||||
let mut socket_reader = unsafe { File::from_raw_fd(socket_fd) };
|
||||
let mut master_writer = unsafe { File::from_raw_fd(master_fd) };
|
||||
|
||||
loop {
|
||||
let mut fd_set = FdSet::new();
|
||||
fd_set.insert(master_fd);
|
||||
fd_set.insert(socket_fd);
|
||||
fd_set.insert(rfd);
|
||||
|
||||
match select(
|
||||
Some(fd_set.highest().unwrap() + 1),
|
||||
&mut fd_set,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
) {
|
||||
Ok(_) => (),
|
||||
Err(e) => {
|
||||
if e == nix::Error::from(nix::errno::Errno::EINTR) {
|
||||
continue;
|
||||
} else {
|
||||
error!(debug_shell_logger, "select error {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fd_set.contains(master_fd) {
|
||||
match io_copy(&mut master_reader, &mut socket_writer) {
|
||||
Ok(0) => {
|
||||
debug!(debug_shell_logger, "master fd closed");
|
||||
break;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => {
|
||||
error!(debug_shell_logger, "read master fd error {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fd_set.contains(socket_fd) {
|
||||
match io_copy(&mut socket_reader, &mut master_writer) {
|
||||
Ok(0) => {
|
||||
debug!(debug_shell_logger, "master fd closed");
|
||||
break;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => {
|
||||
error!(debug_shell_logger, "read master fd error {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fd_set.contains(rfd) {
|
||||
info!(
|
||||
debug_shell_logger,
|
||||
"debug shelll process {} exited", child_pid
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let wait_status = wait::waitpid(child_pid, None);
|
||||
info!(logger, "debug console exit code: {:?}", wait_status);
|
||||
|
||||
// close pipe to exit select loop
|
||||
let _ = close(wfd);
|
||||
}
|
||||
Err(err) => {
|
||||
let msg = format!("fork error: {:?}", err);
|
||||
return Err(ErrorKind::ErrorCode(msg).into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@ -472,8 +632,9 @@ mod tests {
|
||||
let shells_ref = SHELLS.clone();
|
||||
let mut shells = shells_ref.lock().unwrap();
|
||||
shells.clear();
|
||||
let logger = slog_scope::logger();
|
||||
|
||||
let result = setup_debug_console(shells.to_vec(), 0);
|
||||
let result = setup_debug_console(&logger, shells.to_vec(), 0);
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
@ -498,8 +659,9 @@ mod tests {
|
||||
.to_string();
|
||||
|
||||
shells.push(shell);
|
||||
let logger = slog_scope::logger();
|
||||
|
||||
let result = setup_debug_console(shells.to_vec(), 0);
|
||||
let result = setup_debug_console(&logger, shells.to_vec(), 0);
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
|
@ -39,6 +39,7 @@ var version = "@VERSION@"
|
||||
// project-specific command names
|
||||
var envCmd = fmt.Sprintf("%s-env", projectPrefix)
|
||||
var checkCmd = fmt.Sprintf("%s-check", projectPrefix)
|
||||
var execCmd = "exec"
|
||||
|
||||
// project-specific option names
|
||||
var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix)
|
||||
|
208
src/runtime/cli/kata-exec.go
Normal file
208
src/runtime/cli/kata-exec.go
Normal file
@ -0,0 +1,208 @@
|
||||
// Copyright (c) 2017-2019 Intel Corporation
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/console"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
// The buffer size used to specify the buffer for IO streams copy
|
||||
bufSize = 32 << 10
|
||||
|
||||
defaultTimeout = 3 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
bufPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
buffer := make([]byte, bufSize)
|
||||
return &buffer
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
var kataExecCLICommand = cli.Command{
|
||||
Name: execCmd,
|
||||
Usage: "Enter into guest by debug console",
|
||||
Flags: []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "monitor-addr",
|
||||
Usage: "Kata monitor listen address.",
|
||||
},
|
||||
cli.Uint64Flag{
|
||||
Name: "debug-port",
|
||||
Usage: "Port that debug console is listening on.",
|
||||
},
|
||||
},
|
||||
Action: func(context *cli.Context) error {
|
||||
ctx, err := cliContextToContext(context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
span, _ := katautils.Trace(ctx, "exec")
|
||||
defer span.Finish()
|
||||
|
||||
endPoint := context.String("monitor-addr")
|
||||
if endPoint == "" {
|
||||
endPoint = "http://localhost:8090"
|
||||
}
|
||||
|
||||
port := context.Uint64("debug-port")
|
||||
if port == 0 {
|
||||
port = 1026
|
||||
}
|
||||
|
||||
sandboxID := context.Args().Get(0)
|
||||
if sandboxID == "" {
|
||||
return fmt.Errorf("SandboxID not found")
|
||||
}
|
||||
|
||||
conn, err := getConn(endPoint, sandboxID, port)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
con := console.Current()
|
||||
defer con.Reset()
|
||||
|
||||
if err := con.SetRaw(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
iostream := &iostream{
|
||||
conn: conn,
|
||||
exitch: make(chan struct{}),
|
||||
closed: false,
|
||||
}
|
||||
|
||||
ioCopy(iostream, con)
|
||||
|
||||
<-iostream.exitch
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func ioCopy(stream *iostream, con console.Console) {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// stdin
|
||||
go func() {
|
||||
p := bufPool.Get().(*[]byte)
|
||||
defer bufPool.Put(p)
|
||||
io.CopyBuffer(stream, con, *p)
|
||||
}()
|
||||
|
||||
// stdout
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
p := bufPool.Get().(*[]byte)
|
||||
defer bufPool.Put(p)
|
||||
io.CopyBuffer(os.Stdout, stream, *p)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
close(stream.exitch)
|
||||
}
|
||||
|
||||
type iostream struct {
|
||||
conn net.Conn
|
||||
exitch chan struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (s *iostream) Write(data []byte) (n int, err error) {
|
||||
if s.closed {
|
||||
return 0, errors.New("stream closed")
|
||||
}
|
||||
return s.conn.Write(data)
|
||||
}
|
||||
|
||||
func (s *iostream) Close() error {
|
||||
if s.closed {
|
||||
return errors.New("stream closed")
|
||||
}
|
||||
|
||||
err := s.conn.Close()
|
||||
if err == nil {
|
||||
s.closed = true
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *iostream) Read(data []byte) (n int, err error) {
|
||||
if s.closed {
|
||||
return 0, errors.New("stream closed")
|
||||
}
|
||||
|
||||
return s.conn.Read(data)
|
||||
}
|
||||
|
||||
func getConn(endPoint, sandboxID string, port uint64) (net.Conn, error) {
|
||||
shimURL := fmt.Sprintf("%s/agent-url?sandbox=%s", endPoint, sandboxID)
|
||||
resp, err := http.Get(shimURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Failed to get %s: %d", shimURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sock := strings.TrimSuffix(string(data), "\n")
|
||||
addr, err := url.Parse(sock)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// validate more
|
||||
switch addr.Scheme {
|
||||
case clientUtils.VSockSocketScheme:
|
||||
// vsock://31513974:1024
|
||||
shimAddr := clientUtils.VSockSocketScheme + ":" + addr.Host
|
||||
shimAddr = strings.Replace(shimAddr, ":1024", fmt.Sprintf(":%d", port), -1)
|
||||
return clientUtils.VsockDialer(shimAddr, defaultTimeout)
|
||||
|
||||
case clientUtils.HybridVSockScheme:
|
||||
// addr: hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock:1024
|
||||
hvsocket := strings.Split(addr.Path, ":")
|
||||
if len(hvsocket) != 2 {
|
||||
return nil, fmt.Errorf("Invalid hybrid vsock scheme: %s", sock)
|
||||
}
|
||||
|
||||
// hvsock:///run/vc/firecracker/340b412c97bf1375cdda56bfa8f18c8a/root/kata.hvsock
|
||||
shimAddr := fmt.Sprintf("%s:%s:%d", clientUtils.HybridVSockScheme, hvsocket[0], port)
|
||||
return clientUtils.HybridVSockDialer(shimAddr, defaultTimeout)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("schema %s not found", addr.Scheme)
|
||||
}
|
@ -36,6 +36,7 @@ func main() {
|
||||
m := http.NewServeMux()
|
||||
m.Handle("/metrics", http.HandlerFunc(km.ProcessMetricsRequest))
|
||||
m.Handle("/sandboxes", http.HandlerFunc(km.ListSandboxes))
|
||||
m.Handle("/agent-url", http.HandlerFunc(km.GetAgentURL))
|
||||
|
||||
// for debug shim process
|
||||
m.Handle("/debug/vars", http.HandlerFunc(km.ExpvarHandler))
|
||||
|
@ -125,6 +125,7 @@ var runtimeCommands = []cli.Command{
|
||||
// Kata Containers specific extensions
|
||||
kataCheckCLICommand,
|
||||
kataEnvCLICommand,
|
||||
kataExecCLICommand,
|
||||
factoryCLICommand,
|
||||
}
|
||||
|
||||
|
@ -8,6 +8,7 @@ package containerdshim
|
||||
import (
|
||||
"context"
|
||||
"expvar"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
@ -34,6 +35,18 @@ var (
|
||||
shimMgtLog = shimLog.WithField("subsystem", "shim-management")
|
||||
)
|
||||
|
||||
// agentURL returns URL for agent
|
||||
func (s *service) agentURL(w http.ResponseWriter, r *http.Request) {
|
||||
url, err := s.sandbox.GetAgentURL()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprint(w, url)
|
||||
}
|
||||
|
||||
// serveMetrics handle /metrics requests
|
||||
func (s *service) serveMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@ -139,6 +152,7 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec
|
||||
// bind hanlder
|
||||
m := http.NewServeMux()
|
||||
m.Handle("/metrics", http.HandlerFunc(s.serveMetrics))
|
||||
m.Handle("/agent-url", http.HandlerFunc(s.agentURL))
|
||||
s.mountPprofHandle(m, ociSpec)
|
||||
|
||||
// register shim metrics
|
||||
|
@ -10,7 +10,6 @@ import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@ -236,33 +235,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
|
||||
|
||||
// getSandboxMetrics will get sandbox's metrics from shim
|
||||
func (km *KataMonitor) getSandboxMetrics(sandboxID, namespace string) ([]*dto.MetricFamily, error) {
|
||||
socket, err := km.getMonitorAddress(sandboxID, namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
Dial: func(proto, addr string) (conn net.Conn, err error) {
|
||||
return net.Dial("unix", "\x00"+socket)
|
||||
},
|
||||
}
|
||||
|
||||
client := http.Client{
|
||||
Timeout: 3 * time.Second,
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
resp, err := client.Get("http://shim/metrics")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
resp.Body.Close()
|
||||
}()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := km.doGet(sandboxID, namespace, defaultTimeout, "metrics")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -80,6 +80,28 @@ func (km *KataMonitor) initSandboxCache() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAgentURL returns agent URL
|
||||
func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) {
|
||||
sandboxID, err := getSandboxIdFromReq(r)
|
||||
if err != nil {
|
||||
commonServeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
namespace, err := km.getSandboxNamespace(sandboxID)
|
||||
if err != nil {
|
||||
commonServeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := km.doGet(sandboxID, namespace, defaultTimeout, "agent-url")
|
||||
if err != nil {
|
||||
commonServeError(w, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintln(w, string(data))
|
||||
}
|
||||
|
||||
// ListSandboxes list all sandboxes running in Kata
|
||||
func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) {
|
||||
sandboxes := km.getSandboxList()
|
||||
|
@ -12,14 +12,6 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func getSandboxIdFromReq(r *http.Request) (string, error) {
|
||||
sandbox := r.URL.Query().Get("sandbox")
|
||||
if sandbox != "" {
|
||||
return sandbox, nil
|
||||
}
|
||||
return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query())
|
||||
}
|
||||
|
||||
func serveError(w http.ResponseWriter, status int, txt string) {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("X-Go-Pprof", "1")
|
||||
|
81
src/runtime/pkg/kata-monitor/shim_client.go
Normal file
81
src/runtime/pkg/kata-monitor/shim_client.go
Normal file
@ -0,0 +1,81 @@
|
||||
// Copyright (c) 2020 Ant Financial
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
|
||||
package katamonitor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeout = 3 * time.Second
|
||||
)
|
||||
|
||||
func commonServeError(w http.ResponseWriter, status int, err error) {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
if err != nil {
|
||||
fmt.Fprintln(w, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func getSandboxIdFromReq(r *http.Request) (string, error) {
|
||||
sandbox := r.URL.Query().Get("sandbox")
|
||||
if sandbox != "" {
|
||||
return sandbox, nil
|
||||
}
|
||||
return "", fmt.Errorf("sandbox not found in %+v", r.URL.Query())
|
||||
}
|
||||
|
||||
func (km *KataMonitor) buildShimClient(sandboxID, namespace string, timeout time.Duration) (*http.Client, error) {
|
||||
socket, err := km.getMonitorAddress(sandboxID, namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
Dial: func(proto, addr string) (conn net.Conn, err error) {
|
||||
return net.Dial("unix", "\x00"+socket)
|
||||
},
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
if timeout > 0 {
|
||||
client.Timeout = timeout
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (km *KataMonitor) doGet(sandboxID, namespace string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) {
|
||||
client, err := km.buildShimClient(sandboxID, namespace, timeoutInSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
resp.Body.Close()
|
||||
}()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
@ -75,6 +75,7 @@ type VCSandbox interface {
|
||||
|
||||
UpdateRuntimeMetrics() error
|
||||
GetAgentMetrics() (string, error)
|
||||
GetAgentURL() (string, error)
|
||||
}
|
||||
|
||||
// VCContainer is the Container interface
|
||||
|
@ -178,7 +178,7 @@ func parse(sock string) (string, *url.URL, error) {
|
||||
func agentDialer(addr *url.URL) dialer {
|
||||
switch addr.Scheme {
|
||||
case VSockSocketScheme:
|
||||
return vsockDialer
|
||||
return VsockDialer
|
||||
case HybridVSockScheme:
|
||||
return HybridVSockDialer
|
||||
case MockHybridVSockScheme:
|
||||
@ -278,7 +278,7 @@ func commonDialer(timeout time.Duration, dialFunc func() (net.Conn, error), time
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func vsockDialer(sock string, timeout time.Duration) (net.Conn, error) {
|
||||
func VsockDialer(sock string, timeout time.Duration) (net.Conn, error) {
|
||||
cid, port, err := parseGrpcVsockAddr(sock)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -245,7 +245,7 @@ const (
|
||||
// The following example can be used to load two kernel modules with parameters
|
||||
///
|
||||
// annotations:
|
||||
// io.kata-containers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0"
|
||||
// io.katacontainers.config.agent.kernel_modules: "e1000e InterruptThrottleRate=3000,3000,3000 EEE=1; i915 enable_ppgtt=0"
|
||||
//
|
||||
// The first word is considered as the module name and the rest as its parameters.
|
||||
//
|
||||
|
@ -247,3 +247,10 @@ func (s *Sandbox) Stats() (vc.SandboxStats, error) {
|
||||
}
|
||||
return vc.SandboxStats{}, nil
|
||||
}
|
||||
|
||||
func (s *Sandbox) GetAgentURL() (string, error) {
|
||||
if s.GetAgentURLFunc != nil {
|
||||
return s.GetAgentURLFunc()
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
@ -67,6 +67,7 @@ type Sandbox struct {
|
||||
UpdateRuntimeMetricsFunc func() error
|
||||
GetAgentMetricsFunc func() (string, error)
|
||||
StatsFunc func() (vc.SandboxStats, error)
|
||||
GetAgentURLFunc func() (string, error)
|
||||
}
|
||||
|
||||
// Container is a fake Container type used for testing
|
||||
|
@ -2270,3 +2270,7 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
|
||||
func (s *Sandbox) GetOOMEvent() (string, error) {
|
||||
return s.agent.getOOMEvent()
|
||||
}
|
||||
|
||||
func (s *Sandbox) GetAgentURL() (string, error) {
|
||||
return s.agent.getAgentURL()
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user