From f26b8372f852afcedbfc01758a1d56ea9e2a698e Mon Sep 17 00:00:00 2001 From: Andrew Sy Kim Date: Thu, 19 Mar 2020 09:40:08 -0400 Subject: [PATCH] e2e/framework: implement ssh exec internally Signed-off-by: Andrew Sy Kim --- test/e2e/framework/.import-restrictions | 1 - test/e2e/framework/ssh/BUILD | 1 - test/e2e/framework/ssh/ssh.go | 78 +++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions index de7fe003641..d305f1f3f7d 100644 --- a/test/e2e/framework/.import-restrictions +++ b/test/e2e/framework/.import-restrictions @@ -198,7 +198,6 @@ "k8s.io/kubernetes/pkg/security/podsecuritypolicy/util", "k8s.io/kubernetes/pkg/securitycontext", "k8s.io/kubernetes/pkg/serviceaccount", - "k8s.io/kubernetes/pkg/ssh", "k8s.io/kubernetes/pkg/util/async", "k8s.io/kubernetes/pkg/util/bandwidth", "k8s.io/kubernetes/pkg/util/config", diff --git a/test/e2e/framework/ssh/BUILD b/test/e2e/framework/ssh/BUILD index ed880146cbd..0989ab2f96f 100644 --- a/test/e2e/framework/ssh/BUILD +++ b/test/e2e/framework/ssh/BUILD @@ -6,7 +6,6 @@ go_library( importpath = "k8s.io/kubernetes/test/e2e/framework/ssh", visibility = ["//visibility:public"], deps = [ - "//pkg/ssh:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", diff --git a/test/e2e/framework/ssh/ssh.go b/test/e2e/framework/ssh/ssh.go index dbe3a10aef9..adfc7f2f5cf 100644 --- a/test/e2e/framework/ssh/ssh.go +++ b/test/e2e/framework/ssh/ssh.go @@ -20,19 +20,21 @@ import ( "bytes" "context" "fmt" + "io/ioutil" "net" "os" "path/filepath" "time" "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - sshutil "k8s.io/kubernetes/pkg/ssh" e2elog "k8s.io/kubernetes/test/e2e/framework/log" testutils "k8s.io/kubernetes/test/utils" ) @@ -54,7 +56,7 @@ const ( func GetSigner(provider string) (ssh.Signer, error) { // honor a consistent SSH key across all providers if path := os.Getenv("KUBE_SSH_KEY_PATH"); len(path) > 0 { - return sshutil.MakePrivateKeySignerFromFile(path) + return makePrivateKeySignerFromFile(path) } // Select the key itself to use. When implementing more providers here, @@ -93,7 +95,21 @@ func GetSigner(provider string) (ssh.Signer, error) { keyfile = filepath.Join(keydir, keyfile) } - return sshutil.MakePrivateKeySignerFromFile(keyfile) + return makePrivateKeySignerFromFile(keyfile) +} + +func makePrivateKeySignerFromFile(key string) (ssh.Signer, error) { + buffer, err := ioutil.ReadFile(key) + if err != nil { + return nil, fmt.Errorf("error reading SSH key %s: '%v'", key, err) + } + + signer, err := ssh.ParsePrivateKey(buffer) + if err != nil { + return nil, fmt.Errorf("error parsing SSH key: '%v'", err) + } + + return signer, err } // NodeSSHHosts returns SSH-able host names for all schedulable nodes - this @@ -169,7 +185,7 @@ func SSH(cmd, host, provider string) (Result, error) { return result, err } - stdout, stderr, code, err := sshutil.RunSSHCommand(cmd, result.User, host, signer) + stdout, stderr, code, err := runSSHCommand(cmd, result.User, host, signer) result.Stdout = stdout result.Stderr = stderr result.Code = code @@ -177,6 +193,60 @@ func SSH(cmd, host, provider string) (Result, error) { return result, err } +// runSSHCommandViaBastion returns the stdout, stderr, and exit code from running cmd on +// host as specific user, along with any SSH-level error. +func runSSHCommand(cmd, user, host string, signer ssh.Signer) (string, string, int, error) { + if user == "" { + user = os.Getenv("USER") + } + // Setup the config, dial the server, and open a session. + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + client, err := ssh.Dial("tcp", host, config) + if err != nil { + err = wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + fmt.Printf("error dialing %s@%s: '%v', retrying\n", user, host, err) + if client, err = ssh.Dial("tcp", host, config); err != nil { + return false, err + } + return true, nil + }) + } + if err != nil { + return "", "", 0, fmt.Errorf("error getting SSH client to %s@%s: '%v'", user, host, err) + } + defer client.Close() + session, err := client.NewSession() + if err != nil { + return "", "", 0, fmt.Errorf("error creating session to %s@%s: '%v'", user, host, err) + } + defer session.Close() + + // Run the command. + code := 0 + var bout, berr bytes.Buffer + session.Stdout, session.Stderr = &bout, &berr + if err = session.Run(cmd); err != nil { + // Check whether the command failed to run or didn't complete. + if exiterr, ok := err.(*ssh.ExitError); ok { + // If we got an ExitError and the exit code is nonzero, we'll + // consider the SSH itself successful (just that the command run + // errored on the host). + if code = exiterr.ExitStatus(); code != 0 { + err = nil + } + } else { + // Some other kind of error happened (e.g. an IOError); consider the + // SSH unsuccessful. + err = fmt.Errorf("failed running `%s` on %s@%s: '%v'", cmd, user, host, err) + } + } + return bout.String(), berr.String(), code, err +} + // runSSHCommandViaBastion returns the stdout, stderr, and exit code from running cmd on // host as specific user, along with any SSH-level error. It uses an SSH proxy to connect // to bastion, then via that tunnel connects to the remote host. Similar to