Merge pull request #134778 from yt2985/podcertificates-agnhost-2

Add mtlsclient and mtlsserver for the mtls validations

test/images/agnhost/README.md

@@ -367,6 +367,33 @@ Usage:
         [--retry_time <seconds>] [--break_on_expected_content <true_or_false>]
 ```
 
+### mtlsclient
+
+```console
+    kubectl run test-agnhost \
+      --generator=run-pod/v1 \
+      --image=registry.k8s.io/e2e-test-images/agnhost:2.58 \
+      --restart=Always \
+      -- \
+      mtlsclient \
+      --fetch-url=<server-address> \ 
+      --server-trust-bundle=<server-trust-bundle.pem> \ 
+      --client-cred-bundle=<client-cred-bundle.pem>
+```
+
+### mtlsserver
+
+```console
+    kubectl run test-agnhost \
+      --generator=run-pod/v1 \
+      --image=registry.k8s.io/e2e-test-images/agnhost:2.58 \
+      --restart=Always \
+      -- \
+      mtlsserver \
+      --listen=<0.0.0.0:443> \
+      --server-creds=<server-cred-bundle.pem> \ 
+      --spiffe-trust-bundle=<spiffe-trust-bundle.pem> 
+```
 
 ### net
 

test/images/agnhost/VERSION

@@ -1 +1 @@
-2.57
+2.58

test/images/agnhost/agnhost.go

@@ -35,6 +35,8 @@ import (
 	"k8s.io/kubernetes/test/images/agnhost/liveness"
 	logsgen "k8s.io/kubernetes/test/images/agnhost/logs-generator"
 	"k8s.io/kubernetes/test/images/agnhost/mounttest"
+	"k8s.io/kubernetes/test/images/agnhost/mtlsclient"
+	"k8s.io/kubernetes/test/images/agnhost/mtlsserver"
 	"k8s.io/kubernetes/test/images/agnhost/net"
 	"k8s.io/kubernetes/test/images/agnhost/netexec"
 	"k8s.io/kubernetes/test/images/agnhost/nettest"
@@ -92,7 +94,8 @@ func main() {
 	rootCmd.AddCommand(grpchealthchecking.CmdGrpcHealthChecking)
 	rootCmd.AddCommand(vishhstress.CmdStress)
 	rootCmd.AddCommand(podcertificatesigner.CmdPodCertificateSigner)
-
+	rootCmd.AddCommand(mtlsclient.CmdMtlsClient)
+	rootCmd.AddCommand(mtlsserver.CmdMtlsServer)
 	// NOTE(claudiub): Some tests are passing logging related flags, so we need to be able to
 	// accept them. This will also include them in the printed help.
 	code := cli.Run(rootCmd)

test/images/agnhost/mtlsclient/client.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mtlsclient is an agnhost subcommand implementing a client to build
+// the mTLS with the server.
+package mtlsclient
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+)
+
+var CmdMtlsClient = &cobra.Command{
+	Use:   "mtlsclient",
+	Short: "Client is configured to build mTLS with server",
+	Args:  cobra.MaximumNArgs(0),
+	RunE:  main,
+}
+
+var (
+	fetchURL              string
+	serverTrustBundleFile string
+	clientCredBundleFile  string
+)
+
+func init() {
+	CmdMtlsClient.Flags().StringVar(&fetchURL, "fetch-url", "", "server URL to poll")
+	CmdMtlsClient.Flags().StringVar(&serverTrustBundleFile, "server-trust-bundle", "", "File with trust anchors to verify the server certificate")
+	CmdMtlsClient.Flags().StringVar(&clientCredBundleFile, "client-cred-bundle", "", "File with client key and certificate chain")
+}
+
+func main(cmd *cobra.Command, args []string) error {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	if err := flag.Set("logtostderr", "true"); err != nil {
+		return fmt.Errorf("failed to set flags: %w", err)
+	}
+
+	if fetchURL == "" || serverTrustBundleFile == "" || clientCredBundleFile == "" {
+		return fmt.Errorf("missing required flags")
+	}
+
+	if _, err := url.ParseRequestURI(fetchURL); err != nil {
+		return fmt.Errorf("invalid --fetch-url: %w", err)
+	}
+
+	for range time.Tick(10 * time.Second) {
+		if err := pollOnce(); err != nil {
+			klog.Errorf("while sending requests to the server: %v", err)
+		}
+	}
+	return nil
+}
+
+func pollOnce() error {
+	trustBundlePEM, err := os.ReadFile(serverTrustBundleFile)
+	if err != nil {
+		return fmt.Errorf("while reading service trust bundle: %w", err)
+	}
+
+	serverTrustAnchors := x509.NewCertPool()
+	serverTrustAnchors.AppendCertsFromPEM(trustBundlePEM)
+
+	tlsConfig := &tls.Config{
+		RootCAs: serverTrustAnchors,
+	}
+
+	// Load and send client certificates if a bundle file was specified.
+	if clientCredBundleFile != "" {
+		bundlePEM, err := os.ReadFile(clientCredBundleFile)
+		if err != nil {
+			return fmt.Errorf("while reading client credential bundle: %w", err)
+		}
+
+		cert := tls.Certificate{}
+
+		var block *pem.Block
+		rest := bundlePEM
+		for {
+			block, rest = pem.Decode(rest)
+			if block == nil {
+				break
+			}
+
+			switch block.Type {
+			case "PRIVATE KEY":
+				cert.PrivateKey, err = x509.ParsePKCS8PrivateKey(block.Bytes)
+				if err != nil {
+					return fmt.Errorf("while parsing private key from credential bundle: %w", err)
+				}
+			case "CERTIFICATE":
+				cert.Certificate = append(cert.Certificate, block.Bytes)
+			}
+		}
+
+		if cert.PrivateKey == nil {
+			return fmt.Errorf("client credential bundle had no private key")
+		}
+
+		if len(cert.Certificate) == 0 {
+			return fmt.Errorf("client credential bundle had no certificates")
+		}
+
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}
+
+	resp, err := client.Get(fetchURL)
+	if err != nil {
+		return fmt.Errorf("while getting URL: %w", err)
+
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("non-2xx status %d %q", resp.StatusCode, resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("while reading body: %w", err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			log.Printf("while closing response body: %v", err)
+		}
+	}()
+
+	log.Printf("Got response body: %s", string(body))
+	return nil
+}

test/images/agnhost/mtlsserver/server.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mtlsserver is an agnhost subcommand implementing a server to build
+// the mTLS with the client and echo the client identity.
+package mtlsserver
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+	"k8s.io/component-base/logs"
+)
+
+var CmdMtlsServer = &cobra.Command{
+	Use:   "mtlsserver",
+	Short: "Server is configured to build mTLS with client and echo client's spiffe identity",
+	Args:  cobra.MaximumNArgs(0),
+	RunE:  main,
+}
+
+var (
+	listen                string
+	serverCredsFile       string
+	spiffeTrustBundleFile string
+)
+
+func init() {
+	CmdMtlsServer.Flags().StringVar(&listen, "listen", "", "<address>:<port> to listen on")
+	CmdMtlsServer.Flags().StringVar(&serverCredsFile, "server-creds", "", "Credential bundle with the server key and certificate chain")
+	CmdMtlsServer.Flags().StringVar(&spiffeTrustBundleFile, "spiffe-trust-bundle", "", "Trust bundle for verifying client certificates")
+}
+
+func main(cmd *cobra.Command, args []string) error {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	if err := flag.Set("logtostderr", "true"); err != nil {
+		return fmt.Errorf("failed to set flags: %w", err)
+	}
+
+	if listen == "" || serverCredsFile == "" || spiffeTrustBundleFile == "" {
+		return fmt.Errorf("missing required flags")
+	}
+
+	if err := serve(); err != nil {
+		return fmt.Errorf("error while serving: %w", err)
+	}
+	return nil
+}
+
+func serve() error {
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("GET /spiffe-echo", handleGetSPIFFEEcho)
+
+	clientTrustBundlePEM, err := os.ReadFile(spiffeTrustBundleFile)
+	if err != nil {
+		return fmt.Errorf("while reading SPIFFE trust anchors: %w", err)
+	}
+
+	rootPool := x509.NewCertPool()
+	if ok := rootPool.AppendCertsFromPEM(clientTrustBundlePEM); !ok {
+		return fmt.Errorf("failed to append client certs from PEM")
+	}
+
+	// Pre-load server cert to check early if it's readable
+	_, err = tls.LoadX509KeyPair(serverCredsFile, serverCredsFile)
+	if err != nil {
+		return fmt.Errorf("while loading server key pair: %w", err)
+	}
+
+	server := &http.Server{
+		Addr: listen,
+
+		Handler: serveMux,
+
+		TLSConfig: &tls.Config{
+			// Tell the client to send client certs if they have any.  Don't
+			// pass in roots to verify the client certificate, since we expect
+			// multiple types signed by different CAs.  Instead, each endpoint
+			// will do the appropriate client certificate verification.
+			ClientAuth: tls.RequestClientCert,
+		},
+
+		ReadTimeout:    30 * time.Second,
+		WriteTimeout:   30 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	// TODO: Auto-reload the server creds
+	if err := server.ListenAndServeTLS(serverCredsFile, serverCredsFile); err != nil {
+		return fmt.Errorf("while listening: %w", err)
+	}
+
+	return nil
+}
+
+func handleGetSPIFFEEcho(w http.ResponseWriter, r *http.Request) {
+	if len(r.TLS.PeerCertificates) == 0 {
+		http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized)
+		return
+	}
+
+	leafCert := r.TLS.PeerCertificates[0]
+
+	// TODO: Use the trust domain from the leaf certificate to select which
+	// trust bundle to use, to permit federated use cases.
+
+	intermediatePool := x509.NewCertPool()
+	if len(r.TLS.PeerCertificates) > 1 {
+		for _, intermediate := range r.TLS.PeerCertificates[1:] {
+			intermediatePool.AddCert(intermediate)
+		}
+	}
+	rootTrustBundlePEM, err := os.ReadFile(spiffeTrustBundleFile)
+	if err != nil {
+		log.Printf("Error while reading SPIFFE trust anchors: %v", err)
+		http.Error(w, "Internal Error", http.StatusInternalServerError)
+		return
+	}
+	rootPool := x509.NewCertPool()
+	rootPool.AppendCertsFromPEM(rootTrustBundlePEM)
+
+	chains, err := leafCert.Verify(x509.VerifyOptions{
+		Intermediates: intermediatePool,
+		Roots:         rootPool,
+	})
+	if err != nil {
+		log.Printf("Error while verifying SPIFFE client certificate: %v", err)
+		http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized)
+		return
+	}
+	if len(chains) == 0 {
+		log.Print("Client certificate did not chain to any roots")
+		http.Error(w, "SPIFFE client certificate authentication required", http.StatusUnauthorized)
+		return
+	}
+
+	if len(leafCert.URIs) != 1 {
+		log.Printf("SPIFFE client certificate did not have 1 URI SAN, count: %d", len(leafCert.URIs))
+		http.Error(w, "Malformed SPIFFE certificate", http.StatusUnauthorized)
+		return
+	}
+
+	if _, err := w.Write([]byte("Client Identity: " + leafCert.URIs[0].String())); err != nil {
+		log.Printf("Error while writing response: %v", err)
+	}
+}