Merge pull request #101589 from mtaufen/fix-agnhost-ca

agnhost: Try both in-cluster and external discovery

build/dependencies.yaml

@@ -18,7 +18,7 @@ dependencies:
 
   # agnhost: bump this one first
   - name: "agnhost"
-    version: "2.31"
+    version: "2.32"
     refPaths:
     - path: test/images/agnhost/VERSION
       match: \d.\d

test/images/agnhost/VERSION

@@ -1 +1 @@
-2.31
+2.32

test/images/agnhost/agnhost.go

@@ -51,7 +51,7 @@ import (
 func main() {
 	rootCmd := &cobra.Command{
 		Use:     "app",
-		Version: "2.31",
+		Version: "2.32",
 	}
 
 	rootCmd.AddCommand(auditproxy.CmdAuditProxy)

test/images/agnhost/openidmetadata/openidmetadata.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"runtime"
 	"time"
 
@@ -48,32 +49,58 @@ var CmdTestServiceAccountIssuerDiscovery = &cobra.Command{
 }
 
 var (
-	tokenPath          string
+	tokenPath string
-	audience           string
+	audience  string
-	inClusterDiscovery bool
 )
 
 func init() {
 	fs := CmdTestServiceAccountIssuerDiscovery.Flags()
 	fs.StringVar(&tokenPath, "token-path", "", "Path to read service account token from.")
 	fs.StringVar(&audience, "audience", "", "Audience to check on received token.")
-	fs.BoolVar(&inClusterDiscovery, "in-cluster-discovery", false,
-		"Includes the in-cluster bearer token in request headers. "+
-			"Use when validating against API server's discovery endpoints, "+
-			"which require authentication.")
 }
 
 func main(cmd *cobra.Command, args []string) {
-	ctx, err := withOAuth2Client(context.Background())
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	raw, err := gettoken()
 	if err != nil {
 		log.Fatal(err)
 	}
 	log.Print("OK: Got token")
+
+	/*
+		  To support both in-cluster discovery and external (non kube-apiserver)
+		  discovery:
+		  1. Attempt with in-cluster discovery. Only trust Cluster CA.
+		     If pass, exit early, successfully. This attempt includes the bearer
+			 token, so we only trust the Cluster CA to avoid sending tokens to
+			 some external endpoint by accident.
+		  2. If in-cluster discovery doesn't pass, then try again assuming both
+		     discovery doc and JWKS endpoints are external rather than being
+			 served from kube-apiserver. This attempt does not pass the bearer
+			 token at all.
+	*/
+
+	log.Print("validating with in-cluster discovery")
+	inClusterCtx, err := withInClusterOauth2Client(context.Background())
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := validate(inClusterCtx, raw); err == nil {
+		os.Exit(0)
+	} else {
+		log.Print("failed to validate with in-cluster discovery: ", err)
+	}
+
+	log.Print("falling back to validating with external discovery")
+	externalCtx, err := withExternalOAuth2Client(context.Background())
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := validate(externalCtx, raw); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func validate(ctx context.Context, raw string) error {
 	tok, err := jwt.ParseSigned(raw)
 	if err != nil {
 		log.Fatal(err)
@@ -93,7 +120,7 @@ func main(cmd *cobra.Command, args []string) {
 
 	iss, err := oidc.NewProvider(ctx, unsafeClaims.Issuer)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	log.Printf("OK: Constructed OIDC provider for issuer %v", unsafeClaims.Issuer)
 
@@ -102,16 +129,17 @@ func main(cmd *cobra.Command, args []string) {
 		SupportedSigningAlgs: []string{oidc.RS256, oidc.ES256},
 	}).Verify(ctx, raw)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	log.Print("OK: Validated signature on JWT")
 
 	var safeClaims claims
 	if err := validTok.Claims(&safeClaims); err != nil {
-		log.Fatal(err)
+		return err
 	}
 	log.Print("OK: Got valid claims from token!")
 	log.Printf("Full, validated claims: \n%#v", &safeClaims)
+	return nil
 }
 
 type kubeName struct {
@@ -139,42 +167,35 @@ func gettoken() (string, error) {
 	return string(b), err
 }
 
-// withOAuth2Client returns a context that includes an HTTP Client, under the
+func withExternalOAuth2Client(ctx context.Context) (context.Context, error) {
-// oauth2.HTTPClient key. If  --in-cluster-discovery is true, the client will
+	// Use the default http transport with the system root bundle,
-// use the Kubernetes InClusterConfig. Otherwise it will use
-// http.DefaultTransport.
-// The `oidc` library respects the oauth2.HTTPClient context key; if it is set,
-// the library will use the provided http.Client rather than the default
-// HTTP client.
-// This allows us to ensure requests get routed to the API server for
-// --in-cluster-discovery, in a client configured with the appropriate CA.
-func withOAuth2Client(context.Context) (context.Context, error) {
-	// TODO(mtaufen): Someday, might want to change this so that we can test
-	// TokenProjection with an API audience set to the external provider with
-	// requests against external endpoints (in which case we'd send
-	// a different token with a non-Kubernetes audience).
-
-	// By default, use the default http transport with the system root bundle,
 	// since it's validating against the external internet.
-	rt := http.DefaultTransport
+	return context.WithValue(ctx,
-	if inClusterDiscovery {
+		// The `oidc` library respects the oauth2.HTTPClient context key; if it is set,
-		// If in-cluster discovery, then use the in-cluster config so we can
+		// the library will use the provided http.Client rather than the default HTTP client.
-		// authenticate with the API server.
+		oauth2.HTTPClient, &http.Client{
-		cfg, err := rest.InClusterConfig()
+			Transport: http.DefaultTransport,
-		if err != nil {
+		}), nil
-			return nil, err
+}
-		}
+
-		rt, err = rest.TransportFor(cfg)
+func withInClusterOauth2Client(ctx context.Context) (context.Context, error) {
-		if err != nil {
+	// Use the in-cluster config so we can trust and authenticate with kube-apiserver
-			return nil, fmt.Errorf("could not get roundtripper: %v", err)
+	cfg, err := rest.InClusterConfig()
-		}
+	if err != nil {
+		return nil, err
 	}
 
-	ctx := context.WithValue(context.Background(),
+	rt, err := rest.TransportFor(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("could not get roundtripper: %v", err)
+	}
+
+	return context.WithValue(ctx,
+		// The `oidc` library respects the oauth2.HTTPClient context key; if it is set,
+		// the library will use the provided http.Client rather than the default HTTP client.
 		oauth2.HTTPClient, &http.Client{
 			Transport: rt,
-		})
+		}), nil
-	return ctx, nil
 }
 
 // DNS can be available sometime after the container starts due to the way