From 9e5b6ad7488cc89e768f76d83866bb6c22337691 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Wed, 22 Oct 2025 17:14:02 +0200 Subject: [PATCH] fix error checks --- test/images/agnhost/README.md | 27 ++++ test/images/agnhost/VERSION | 2 +- test/images/agnhost/agnhost.go | 5 +- test/images/agnhost/mtlsclient/client.go | 161 ++++++++++++++++++++++ test/images/agnhost/mtlsserver/server.go | 167 +++++++++++++++++++++++ 5 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 test/images/agnhost/mtlsclient/client.go create mode 100644 test/images/agnhost/mtlsserver/server.go diff --git a/test/images/agnhost/README.md b/test/images/agnhost/README.md index 0494ec162d3..045e842893b 100644 --- a/test/images/agnhost/README.md +++ b/test/images/agnhost/README.md @@ -367,6 +367,33 @@ Usage: [--retry_time ] [--break_on_expected_content ] ``` +### mtlsclient + +```console + kubectl run test-agnhost \ + --generator=run-pod/v1 \ + --image=registry.k8s.io/e2e-test-images/agnhost:2.58 \ + --restart=Always \ + -- \ + mtlsclient \ + --fetch-url= \ + --server-trust-bundle= \ + --client-cred-bundle= +``` + +### mtlsserver + +```console + kubectl run test-agnhost \ + --generator=run-pod/v1 \ + --image=registry.k8s.io/e2e-test-images/agnhost:2.58 \ + --restart=Always \ + -- \ + mtlsserver \ + --listen=<0.0.0.0:443> \ + --server-creds= \ + --spiffe-trust-bundle= +``` ### net diff --git a/test/images/agnhost/VERSION b/test/images/agnhost/VERSION index 7521537d966..995e96d135c 100644 --- a/test/images/agnhost/VERSION +++ b/test/images/agnhost/VERSION @@ -1 +1 @@ -2.57 +2.58 diff --git a/test/images/agnhost/agnhost.go b/test/images/agnhost/agnhost.go index e091c4b292d..5bef6e8b40b 100644 --- a/test/images/agnhost/agnhost.go +++ b/test/images/agnhost/agnhost.go @@ -35,6 +35,8 @@ import ( "k8s.io/kubernetes/test/images/agnhost/liveness" logsgen "k8s.io/kubernetes/test/images/agnhost/logs-generator" "k8s.io/kubernetes/test/images/agnhost/mounttest" + "k8s.io/kubernetes/test/images/agnhost/mtlsclient" + "k8s.io/kubernetes/test/images/agnhost/mtlsserver" "k8s.io/kubernetes/test/images/agnhost/net" "k8s.io/kubernetes/test/images/agnhost/netexec" "k8s.io/kubernetes/test/images/agnhost/nettest" @@ -92,7 +94,8 @@ func main() { rootCmd.AddCommand(grpchealthchecking.CmdGrpcHealthChecking) rootCmd.AddCommand(vishhstress.CmdStress) rootCmd.AddCommand(podcertificatesigner.CmdPodCertificateSigner) - + rootCmd.AddCommand(mtlsclient.CmdMtlsClient) + rootCmd.AddCommand(mtlsserver.CmdMtlsServer) // NOTE(claudiub): Some tests are passing logging related flags, so we need to be able to // accept them. This will also include them in the printed help. code := cli.Run(rootCmd) diff --git a/test/images/agnhost/mtlsclient/client.go b/test/images/agnhost/mtlsclient/client.go new file mode 100644 index 00000000000..e8b3055453f --- /dev/null +++ b/test/images/agnhost/mtlsclient/client.go @@ -0,0 +1,161 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mtlsclient is an agnhost subcommand implementing a client to build +// the mTLS with the server. +package mtlsclient + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "time" + + "github.com/spf13/cobra" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" +) + +var CmdMtlsClient = &cobra.Command{ + Use: "mtlsclient", + Short: "Client is configured to build mTLS with server", + Args: cobra.MaximumNArgs(0), + RunE: main, +} + +var ( + fetchURL string + serverTrustBundleFile string + clientCredBundleFile string +) + +func init() { + CmdMtlsClient.Flags().StringVar(&fetchURL, "fetch-url", "", "server URL to poll") + CmdMtlsClient.Flags().StringVar(&serverTrustBundleFile, "server-trust-bundle", "", "File with trust anchors to verify the server certificate") + CmdMtlsClient.Flags().StringVar(&clientCredBundleFile, "client-cred-bundle", "", "File with client key and certificate chain") +} + +func main(cmd *cobra.Command, args []string) error { + logs.InitLogs() + defer logs.FlushLogs() + if err := flag.Set("logtostderr", "true"); err != nil { + return fmt.Errorf("failed to set flags: %w", err) + } + + if fetchURL == "" || serverTrustBundleFile == "" || clientCredBundleFile == "" { + return fmt.Errorf("missing required flags") + } + + if _, err := url.ParseRequestURI(fetchURL); err != nil { + return fmt.Errorf("invalid --fetch-url: %w", err) + } + + for range time.Tick(10 * time.Second) { + if err := pollOnce(); err != nil { + klog.Errorf("while sending requests to the server: %v", err) + } + } + return nil +} + +func pollOnce() error { + trustBundlePEM, err := os.ReadFile(serverTrustBundleFile) + if err != nil { + return fmt.Errorf("while reading service trust bundle: %w", err) + } + + serverTrustAnchors := x509.NewCertPool() + serverTrustAnchors.AppendCertsFromPEM(trustBundlePEM) + + tlsConfig := &tls.Config{ + RootCAs: serverTrustAnchors, + } + + // Load and send client certificates if a bundle file was specified. + if clientCredBundleFile != "" { + bundlePEM, err := os.ReadFile(clientCredBundleFile) + if err != nil { + return fmt.Errorf("while reading client credential bundle: %w", err) + } + + cert := tls.Certificate{} + + var block *pem.Block + rest := bundlePEM + for { + block, rest = pem.Decode(rest) + if block == nil { + break + } + + switch block.Type { + case "PRIVATE KEY": + cert.PrivateKey, err = x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return fmt.Errorf("while parsing private key from credential bundle: %w", err) + } + case "CERTIFICATE": + cert.Certificate = append(cert.Certificate, block.Bytes) + } + } + + if cert.PrivateKey == nil { + return fmt.Errorf("client credential bundle had no private key") + } + + if len(cert.Certificate) == 0 { + return fmt.Errorf("client credential bundle had no certificates") + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + + resp, err := client.Get(fetchURL) + if err != nil { + return fmt.Errorf("while getting URL: %w", err) + + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("non-2xx status %d %q", resp.StatusCode, resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("while reading body: %w", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Printf("while closing response body: %v", err) + } + }() + + log.Printf("Got response body: %s", string(body)) + return nil +} diff --git a/test/images/agnhost/mtlsserver/server.go b/test/images/agnhost/mtlsserver/server.go new file mode 100644 index 00000000000..dcf9e0e6d94 --- /dev/null +++ b/test/images/agnhost/mtlsserver/server.go @@ -0,0 +1,167 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mtlsserver is an agnhost subcommand implementing a server to build +// the mTLS with the client and echo the client identity. +package mtlsserver + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "net/http" + "os" + "time" + + "github.com/spf13/cobra" + "k8s.io/component-base/logs" +) + +var CmdMtlsServer = &cobra.Command{ + Use: "mtlsserver", + Short: "Server is configured to build mTLS with client and echo client's spiffe identity", + Args: cobra.MaximumNArgs(0), + RunE: main, +} + +var ( + listen string + serverCredsFile string + spiffeTrustBundleFile string +) + +func init() { + CmdMtlsServer.Flags().StringVar(&listen, "listen", "", "
: to listen on") + CmdMtlsServer.Flags().StringVar(&serverCredsFile, "server-creds", "", "Credential bundle with the server key and certificate chain") + CmdMtlsServer.Flags().StringVar(&spiffeTrustBundleFile, "spiffe-trust-bundle", "", "Trust bundle for verifying client certificates") +} + +func main(cmd *cobra.Command, args []string) error { + logs.InitLogs() + defer logs.FlushLogs() + if err := flag.Set("logtostderr", "true"); err != nil { + return fmt.Errorf("failed to set flags: %w", err) + } + + if listen == "" || serverCredsFile == "" || spiffeTrustBundleFile == "" { + return fmt.Errorf("missing required flags") + } + + if err := serve(); err != nil { + return fmt.Errorf("error while serving: %w", err) + } + return nil +} + +func serve() error { + serveMux := http.NewServeMux() + serveMux.HandleFunc("GET /spiffe-echo", handleGetSPIFFEEcho) + + clientTrustBundlePEM, err := os.ReadFile(spiffeTrustBundleFile) + if err != nil { + return fmt.Errorf("while reading SPIFFE trust anchors: %w", err) + } + + rootPool := x509.NewCertPool() + if ok := rootPool.AppendCertsFromPEM(clientTrustBundlePEM); !ok { + return fmt.Errorf("failed to append client certs from PEM") + } + + // Pre-load server cert to check early if it's readable + _, err = tls.LoadX509KeyPair(serverCredsFile, serverCredsFile) + if err != nil { + return fmt.Errorf("while loading server key pair: %w", err) + } + + server := &http.Server{ + Addr: listen, + + Handler: serveMux, + + TLSConfig: &tls.Config{ + // Tell the client to send client certs if they have any. Don't + // pass in roots to verify the client certificate, since we expect + // multiple types signed by different CAs. Instead, each endpoint + // will do the appropriate client certificate verification. + ClientAuth: tls.RequestClientCert, + }, + + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + // TODO: Auto-reload the server creds + if err := server.ListenAndServeTLS(serverCredsFile, serverCredsFile); err != nil { + return fmt.Errorf("while listening: %w", err) + } + + return nil +} + +func handleGetSPIFFEEcho(w http.ResponseWriter, r *http.Request) { + if len(r.TLS.PeerCertificates) == 0 { + http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized) + return + } + + leafCert := r.TLS.PeerCertificates[0] + + // TODO: Use the trust domain from the leaf certificate to select which + // trust bundle to use, to permit federated use cases. + + intermediatePool := x509.NewCertPool() + if len(r.TLS.PeerCertificates) > 1 { + for _, intermediate := range r.TLS.PeerCertificates[1:] { + intermediatePool.AddCert(intermediate) + } + } + rootTrustBundlePEM, err := os.ReadFile(spiffeTrustBundleFile) + if err != nil { + log.Printf("Error while reading SPIFFE trust anchors: %v", err) + http.Error(w, "Internal Error", http.StatusInternalServerError) + return + } + rootPool := x509.NewCertPool() + rootPool.AppendCertsFromPEM(rootTrustBundlePEM) + + chains, err := leafCert.Verify(x509.VerifyOptions{ + Intermediates: intermediatePool, + Roots: rootPool, + }) + if err != nil { + log.Printf("Error while verifying SPIFFE client certificate: %v", err) + http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized) + return + } + if len(chains) == 0 { + log.Print("Client certificate did not chain to any roots") + http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized) + return + } + + if len(leafCert.URIs) != 1 { + log.Printf("SPIFFE client certificate did not have 1 URI SAN, count: %d", len(leafCert.URIs)) + http.Error(w, "Malformed SPIFFE certificate", http.StatusUnauthorized) + return + } + + if _, err := w.Write([]byte("Client Identity: " + leafCert.URIs[0].String())); err != nil { + log.Printf("Error while writing response: %v", err) + } +}