Merge pull request #116645 from mborsz/kubemark

Add missing trace dependency to kubemark

cmd/kubemark/app/hollow_node.go

@@ -0,0 +1,302 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"context"
+	"errors"
+	goflag "flag"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	internalapi "k8s.io/cri-api/pkg/apis"
+	"k8s.io/klog/v2"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/events"
+	cliflag "k8s.io/component-base/cli/flag"
+	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
+	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
+	"k8s.io/component-base/version"
+	"k8s.io/component-base/version/verflag"
+	fakesysctl "k8s.io/component-helpers/node/util/sysctl/testing"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/cluster/ports"
+	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
+	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
+	"k8s.io/kubernetes/pkg/kubelet/cm"
+	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
+	fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
+	"k8s.io/kubernetes/pkg/kubemark"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+type hollowNodeConfig struct {
+	KubeconfigPath          string
+	BootstrapKubeconfigPath string
+	CertDirectory           string
+	KubeletPort             int
+	KubeletReadOnlyPort     int
+	Morph                   string
+	NodeName                string
+	ServerPort              int
+	ContentType             string
+	UseRealProxier          bool
+	ProxierSyncPeriod       time.Duration
+	ProxierMinSyncPeriod    time.Duration
+	NodeLabels              map[string]string
+	RegisterWithTaints      []v1.Taint
+	MaxPods                 int
+	ExtendedResources       map[string]string
+	UseHostImageService     bool
+}
+
+const (
+	maxPods     = 110
+	podsPerCore = 0
+)
+
+// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
+// and make the config driven.
+var knownMorphs = sets.NewString("kubelet", "proxy")
+
+func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
+	fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
+	fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
+	fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
+	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
+	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
+	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
+	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
+	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
+	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
+	fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.")
+	fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.")
+	bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
+	fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
+	fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
+	fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
+	bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
+	fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
+	fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
+}
+
+func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
+	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
+	if err != nil {
+		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
+	}
+	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
+	}
+	config.ContentType = c.ContentType
+	config.QPS = 10
+	config.Burst = 20
+	return config, nil
+}
+
+func (c *hollowNodeConfig) bootstrapClientConfig() error {
+	if c.BootstrapKubeconfigPath != "" {
+		return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
+	}
+	return nil
+}
+
+func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
+	return &kubemark.HollowKubeletOptions{
+		NodeName:            c.NodeName,
+		KubeletPort:         c.KubeletPort,
+		KubeletReadOnlyPort: c.KubeletReadOnlyPort,
+		MaxPods:             c.MaxPods,
+		PodsPerCore:         podsPerCore,
+		NodeLabels:          c.NodeLabels,
+		RegisterWithTaints:  c.RegisterWithTaints,
+	}
+}
+
+// NewHollowNodeCommand creates a *cobra.Command object with default parameters
+func NewHollowNodeCommand() *cobra.Command {
+	s := &hollowNodeConfig{
+		NodeLabels:        make(map[string]string),
+		ExtendedResources: make(map[string]string),
+	}
+
+	cmd := &cobra.Command{
+		Use:  "kubemark",
+		Long: "kubemark",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			verflag.PrintAndExitIfRequested()
+			cliflag.PrintFlags(cmd.Flags())
+			return run(s)
+		},
+		Args: func(cmd *cobra.Command, args []string) error {
+			for _, arg := range args {
+				if len(arg) > 0 {
+					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
+				}
+			}
+			return nil
+		},
+	}
+
+	fs := cmd.Flags()
+	fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
+	s.addFlags(fs)
+
+	return cmd
+}
+
+func run(config *hollowNodeConfig) error {
+	// To help debugging, immediately log version and print flags.
+	klog.Infof("Version: %+v", version.Get())
+
+	if !knownMorphs.Has(config.Morph) {
+		return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
+	}
+
+	// create a client to communicate with API server.
+	err := config.bootstrapClientConfig()
+	if err != nil {
+		return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
+	}
+	clientConfig, err := config.createClientConfigFromFile()
+	if err != nil {
+		return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
+	}
+
+	if config.Morph == "kubelet" {
+		clientConfig.UserAgent = "hollow-kubelet"
+		client, err := clientset.NewForConfig(clientConfig)
+		if err != nil {
+			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
+		}
+
+		f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
+
+		heartbeatClientConfig := *clientConfig
+		heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
+		// The timeout is the minimum of the lease duration and status update frequency
+		leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
+		if heartbeatClientConfig.Timeout > leaseTimeout {
+			heartbeatClientConfig.Timeout = leaseTimeout
+		}
+
+		heartbeatClientConfig.QPS = float32(-1)
+		heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
+		if err != nil {
+			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
+		}
+
+		cadvisorInterface := &cadvisortest.Fake{
+			NodeName: config.NodeName,
+		}
+
+		var containerManager cm.ContainerManager
+		if config.ExtendedResources != nil {
+			extendedResources := v1.ResourceList{}
+			for k, v := range config.ExtendedResources {
+				extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
+			}
+
+			containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
+		} else {
+			containerManager = cm.NewStubContainerManager()
+		}
+
+		endpoint, err := fakeremote.GenerateEndpoint()
+		if err != nil {
+			return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
+		}
+		fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
+		if err = fakeRemoteRuntime.Start(endpoint); err != nil {
+			return fmt.Errorf("Failed to start fake runtime, error: %w", err)
+		}
+		defer fakeRemoteRuntime.Stop()
+		runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
+		if err != nil {
+			return fmt.Errorf("Failed to init runtime service, error: %w", err)
+		}
+
+		var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
+		if config.UseHostImageService {
+			imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
+			if err != nil {
+				return fmt.Errorf("Failed to init image service, error: %w", err)
+			}
+		}
+
+		hollowKubelet := kubemark.NewHollowKubelet(
+			f, c,
+			client,
+			heartbeatClient,
+			cadvisorInterface,
+			imageService,
+			runtimeService,
+			containerManager,
+		)
+		hollowKubelet.Run()
+	}
+
+	if config.Morph == "proxy" {
+		clientConfig.UserAgent = "hollow-proxy"
+
+		client, err := clientset.NewForConfig(clientConfig)
+		if err != nil {
+			return fmt.Errorf("Failed to create API Server client, error: %w", err)
+		}
+		iptInterface := fakeiptables.NewFake()
+		sysctl := fakesysctl.NewFake()
+		execer := &fakeexec.FakeExec{
+			LookPathFunc: func(_ string) (string, error) { return "", errors.New("fake execer") },
+		}
+		eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
+		recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
+
+		hollowProxy, err := kubemark.NewHollowProxyOrDie(
+			config.NodeName,
+			client,
+			client.CoreV1(),
+			iptInterface,
+			sysctl,
+			execer,
+			eventBroadcaster,
+			recorder,
+			config.UseRealProxier,
+			config.ProxierSyncPeriod,
+			config.ProxierMinSyncPeriod,
+		)
+		if err != nil {
+			return fmt.Errorf("Failed to create hollowProxy instance, error: %w", err)
+		}
+		return hollowProxy.Run()
+	}
+
+	return nil
+}

cmd/kubemark/app/hollow_node_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+const fakeKubeconfig = `
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    insecure-skip-tls-verify: true
+    server: %s
+  name: default
+contexts:
+- context:
+    cluster: default
+    user: default
+  name: default
+current-context: default
+users:
+- name: default
+  user:
+    username: config
+`
+
+// TestHollowNode is a naive test that attempts to start hollow node and checks if it's not crashing.
+// Such test is sufficient to detect e.g. missing kubelet dependencies that are not added in
+// pkg/kubemark/hollow_kubelet.go.
+func TestHollowNode(t *testing.T) {
+	// temp dir
+	tmpDir, err := os.MkdirTemp("", "hollow-node")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// https server
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.WriteHeader(200)
+		w.Write([]byte(`ok`))
+	}))
+	defer server.Close()
+
+	kubeconfigPath := filepath.Join(tmpDir, "config.kubeconfig")
+	if err := os.WriteFile(kubeconfigPath, []byte(fmt.Sprintf(fakeKubeconfig, server.URL)), os.FileMode(0600)); err != nil {
+		t.Fatal(err)
+	}
+
+	for morph := range knownMorphs {
+		morph := morph
+		t.Run(morph, func(t *testing.T) {
+			s := &hollowNodeConfig{
+				KubeconfigPath: kubeconfigPath,
+				Morph:          morph,
+			}
+			errCh := make(chan error)
+			go func() {
+				data, err := os.ReadFile(kubeconfigPath)
+				t.Logf("read %d, err=%v\n", len(data), err)
+				errCh <- run(s)
+			}()
+
+			select {
+			case err := <-errCh:
+				t.Fatalf("Run finished unexpectedly with error: %v", err)
+			case <-time.After(3 * time.Second):
+				t.Logf("Morph %q hasn't crashed for 3s. Calling success.", morph)
+			}
+		})
+	}
+}

cmd/kubemark/hollow-node.go

@@ -17,294 +17,14 @@ limitations under the License.
 package main
 
 import (
-	"context"
-	"errors"
-	goflag "flag"
-	"fmt"
 	"os"
-	"time"
 
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	oteltrace "go.opentelemetry.io/otel/trace"
-	internalapi "k8s.io/cri-api/pkg/apis"
-	"k8s.io/klog/v2"
-
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/sets"
-	clientset "k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/events"
 	"k8s.io/component-base/cli"
-	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/kubernetes/cmd/kubemark/app"
-	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
-	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
-	"k8s.io/component-base/version"
-	"k8s.io/component-base/version/verflag"
-	fakesysctl "k8s.io/component-helpers/node/util/sysctl/testing"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
-	"k8s.io/kubernetes/pkg/cluster/ports"
-	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
-	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
-	"k8s.io/kubernetes/pkg/kubelet/cm"
-	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
-	fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
-	"k8s.io/kubernetes/pkg/kubemark"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
-	fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
-	fakeexec "k8s.io/utils/exec/testing"
 )
 
-type hollowNodeConfig struct {
-	KubeconfigPath          string
-	BootstrapKubeconfigPath string
-	CertDirectory           string
-	KubeletPort             int
-	KubeletReadOnlyPort     int
-	Morph                   string
-	NodeName                string
-	ServerPort              int
-	ContentType             string
-	UseRealProxier          bool
-	ProxierSyncPeriod       time.Duration
-	ProxierMinSyncPeriod    time.Duration
-	NodeLabels              map[string]string
-	RegisterWithTaints      []v1.Taint
-	MaxPods                 int
-	ExtendedResources       map[string]string
-	UseHostImageService     bool
-}
-
-const (
-	maxPods     = 110
-	podsPerCore = 0
-)
-
-// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
-// and make the config driven.
-var knownMorphs = sets.NewString("kubelet", "proxy")
-
-func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
-	fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
-	fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
-	fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
-	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
-	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
-	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
-	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
-	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
-	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
-	fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.")
-	fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.")
-	bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
-	fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
-	fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
-	fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
-	bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
-	fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
-	fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
-}
-
-func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
-	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
-	if err != nil {
-		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
-	}
-	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
-	if err != nil {
-		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
-	}
-	config.ContentType = c.ContentType
-	config.QPS = 10
-	config.Burst = 20
-	return config, nil
-}
-
-func (c *hollowNodeConfig) bootstrapClientConfig() error {
-	if c.BootstrapKubeconfigPath != "" {
-		return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
-	}
-	return nil
-}
-
-func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
-	return &kubemark.HollowKubeletOptions{
-		NodeName:            c.NodeName,
-		KubeletPort:         c.KubeletPort,
-		KubeletReadOnlyPort: c.KubeletReadOnlyPort,
-		MaxPods:             c.MaxPods,
-		PodsPerCore:         podsPerCore,
-		NodeLabels:          c.NodeLabels,
-		RegisterWithTaints:  c.RegisterWithTaints,
-	}
-}
-
 func main() {
-	command := newHollowNodeCommand()
+	command := app.NewHollowNodeCommand()
 	code := cli.Run(command)
 	os.Exit(code)
 }
-
-// newControllerManagerCommand creates a *cobra.Command object with default parameters
-func newHollowNodeCommand() *cobra.Command {
-	s := &hollowNodeConfig{
-		NodeLabels:        make(map[string]string),
-		ExtendedResources: make(map[string]string),
-	}
-
-	cmd := &cobra.Command{
-		Use:  "kubemark",
-		Long: "kubemark",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			verflag.PrintAndExitIfRequested()
-			return run(cmd, s)
-		},
-		Args: func(cmd *cobra.Command, args []string) error {
-			for _, arg := range args {
-				if len(arg) > 0 {
-					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
-				}
-			}
-			return nil
-		},
-	}
-
-	fs := cmd.Flags()
-	fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
-	s.addFlags(fs)
-
-	return cmd
-}
-
-func run(cmd *cobra.Command, config *hollowNodeConfig) error {
-	// To help debugging, immediately log version and print flags.
-	klog.Infof("Version: %+v", version.Get())
-	cliflag.PrintFlags(cmd.Flags())
-
-	if !knownMorphs.Has(config.Morph) {
-		return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
-	}
-
-	// create a client to communicate with API server.
-	err := config.bootstrapClientConfig()
-	if err != nil {
-		return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
-	}
-	clientConfig, err := config.createClientConfigFromFile()
-	if err != nil {
-		return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
-	}
-
-	if config.Morph == "kubelet" {
-		clientConfig.UserAgent = "hollow-kubelet"
-		client, err := clientset.NewForConfig(clientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
-		}
-
-		f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
-
-		heartbeatClientConfig := *clientConfig
-		heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
-		// The timeout is the minimum of the lease duration and status update frequency
-		leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
-		if heartbeatClientConfig.Timeout > leaseTimeout {
-			heartbeatClientConfig.Timeout = leaseTimeout
-		}
-
-		heartbeatClientConfig.QPS = float32(-1)
-		heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
-		}
-
-		cadvisorInterface := &cadvisortest.Fake{
-			NodeName: config.NodeName,
-		}
-
-		var containerManager cm.ContainerManager
-		if config.ExtendedResources != nil {
-			extendedResources := v1.ResourceList{}
-			for k, v := range config.ExtendedResources {
-				extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
-			}
-
-			containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
-		} else {
-			containerManager = cm.NewStubContainerManager()
-		}
-
-		endpoint, err := fakeremote.GenerateEndpoint()
-		if err != nil {
-			return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
-		}
-		fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
-		if err = fakeRemoteRuntime.Start(endpoint); err != nil {
-			return fmt.Errorf("Failed to start fake runtime, error: %w", err)
-		}
-		defer fakeRemoteRuntime.Stop()
-		runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
-		if err != nil {
-			return fmt.Errorf("Failed to init runtime service, error: %w", err)
-		}
-
-		var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
-		if config.UseHostImageService {
-			imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
-			if err != nil {
-				return fmt.Errorf("Failed to init image service, error: %w", err)
-			}
-		}
-
-		hollowKubelet := kubemark.NewHollowKubelet(
-			f, c,
-			client,
-			heartbeatClient,
-			cadvisorInterface,
-			imageService,
-			runtimeService,
-			containerManager,
-		)
-		hollowKubelet.Run()
-	}
-
-	if config.Morph == "proxy" {
-		clientConfig.UserAgent = "hollow-proxy"
-
-		client, err := clientset.NewForConfig(clientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create API Server client, error: %w", err)
-		}
-		iptInterface := fakeiptables.NewFake()
-		sysctl := fakesysctl.NewFake()
-		execer := &fakeexec.FakeExec{
-			LookPathFunc: func(_ string) (string, error) { return "", errors.New("fake execer") },
-		}
-		eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
-		recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
-
-		hollowProxy, err := kubemark.NewHollowProxyOrDie(
-			config.NodeName,
-			client,
-			client.CoreV1(),
-			iptInterface,
-			sysctl,
-			execer,
-			eventBroadcaster,
-			recorder,
-			config.UseRealProxier,
-			config.ProxierSyncPeriod,
-			config.ProxierMinSyncPeriod,
-		)
-		if err != nil {
-			return fmt.Errorf("Failed to create hollowProxy instance, error: %w", err)
-		}
-		return hollowProxy.Run()
-	}
-
-	return nil
-}

pkg/kubemark/hollow_kubelet.go

@@ -20,12 +20,14 @@ import (
 	"fmt"
 	"time"
 
+	"go.opentelemetry.io/otel/trace"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/mount-utils"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/record"
 	internalapi "k8s.io/cri-api/pkg/apis"
 	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
@@ -111,6 +113,8 @@ func NewHollowKubelet(
 		Subpather:                &subpath.FakeSubpath{},
 		HostUtil:                 hostutil.NewFakeHostUtil(nil),
 		PodStartupLatencyTracker: kubeletutil.NewPodStartupLatencyTracker(),
+		TracerProvider:           trace.NewNoopTracerProvider(),
+		Recorder:                 &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
 	}
 
 	return &HollowKubelet{