mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-19 09:52:49 +00:00
Merge pull request #116645 from mborsz/kubemark
Add missing trace dependency to kubemark
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
e6f3e3dddd
302
cmd/kubemark/app/hollow_node.go
Normal file
302
cmd/kubemark/app/hollow_node.go
Normal file
@ -0,0 +1,302 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
goflag "flag"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
oteltrace "go.opentelemetry.io/otel/trace"
|
||||
internalapi "k8s.io/cri-api/pkg/apis"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/tools/events"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
|
||||
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
|
||||
"k8s.io/component-base/version"
|
||||
"k8s.io/component-base/version/verflag"
|
||||
fakesysctl "k8s.io/component-helpers/node/util/sysctl/testing"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/cluster/ports"
|
||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||
"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cri/remote"
|
||||
fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
|
||||
"k8s.io/kubernetes/pkg/kubemark"
|
||||
utilflag "k8s.io/kubernetes/pkg/util/flag"
|
||||
fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
|
||||
fakeexec "k8s.io/utils/exec/testing"
|
||||
)
|
||||
|
||||
type hollowNodeConfig struct {
|
||||
KubeconfigPath string
|
||||
BootstrapKubeconfigPath string
|
||||
CertDirectory string
|
||||
KubeletPort int
|
||||
KubeletReadOnlyPort int
|
||||
Morph string
|
||||
NodeName string
|
||||
ServerPort int
|
||||
ContentType string
|
||||
UseRealProxier bool
|
||||
ProxierSyncPeriod time.Duration
|
||||
ProxierMinSyncPeriod time.Duration
|
||||
NodeLabels map[string]string
|
||||
RegisterWithTaints []v1.Taint
|
||||
MaxPods int
|
||||
ExtendedResources map[string]string
|
||||
UseHostImageService bool
|
||||
}
|
||||
|
||||
const (
|
||||
maxPods = 110
|
||||
podsPerCore = 0
|
||||
)
|
||||
|
||||
// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
|
||||
// and make the config driven.
|
||||
var knownMorphs = sets.NewString("kubelet", "proxy")
|
||||
|
||||
func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
|
||||
fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
|
||||
fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
|
||||
fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
|
||||
fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
|
||||
fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
|
||||
fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
|
||||
fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
|
||||
fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
|
||||
fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
|
||||
fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
|
||||
fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.")
|
||||
fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.")
|
||||
bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
|
||||
fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
|
||||
fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
|
||||
fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
|
||||
bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
|
||||
fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
|
||||
fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
|
||||
clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
|
||||
}
|
||||
config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
|
||||
}
|
||||
config.ContentType = c.ContentType
|
||||
config.QPS = 10
|
||||
config.Burst = 20
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) bootstrapClientConfig() error {
|
||||
if c.BootstrapKubeconfigPath != "" {
|
||||
return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
|
||||
return &kubemark.HollowKubeletOptions{
|
||||
NodeName: c.NodeName,
|
||||
KubeletPort: c.KubeletPort,
|
||||
KubeletReadOnlyPort: c.KubeletReadOnlyPort,
|
||||
MaxPods: c.MaxPods,
|
||||
PodsPerCore: podsPerCore,
|
||||
NodeLabels: c.NodeLabels,
|
||||
RegisterWithTaints: c.RegisterWithTaints,
|
||||
}
|
||||
}
|
||||
|
||||
// NewHollowNodeCommand creates a *cobra.Command object with default parameters
|
||||
func NewHollowNodeCommand() *cobra.Command {
|
||||
s := &hollowNodeConfig{
|
||||
NodeLabels: make(map[string]string),
|
||||
ExtendedResources: make(map[string]string),
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "kubemark",
|
||||
Long: "kubemark",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
verflag.PrintAndExitIfRequested()
|
||||
cliflag.PrintFlags(cmd.Flags())
|
||||
return run(s)
|
||||
},
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
for _, arg := range args {
|
||||
if len(arg) > 0 {
|
||||
return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
fs := cmd.Flags()
|
||||
fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
|
||||
s.addFlags(fs)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func run(config *hollowNodeConfig) error {
|
||||
// To help debugging, immediately log version and print flags.
|
||||
klog.Infof("Version: %+v", version.Get())
|
||||
|
||||
if !knownMorphs.Has(config.Morph) {
|
||||
return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
|
||||
}
|
||||
|
||||
// create a client to communicate with API server.
|
||||
err := config.bootstrapClientConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
|
||||
}
|
||||
clientConfig, err := config.createClientConfigFromFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
if config.Morph == "kubelet" {
|
||||
clientConfig.UserAgent = "hollow-kubelet"
|
||||
client, err := clientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
|
||||
|
||||
heartbeatClientConfig := *clientConfig
|
||||
heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
|
||||
// The timeout is the minimum of the lease duration and status update frequency
|
||||
leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
|
||||
if heartbeatClientConfig.Timeout > leaseTimeout {
|
||||
heartbeatClientConfig.Timeout = leaseTimeout
|
||||
}
|
||||
|
||||
heartbeatClientConfig.QPS = float32(-1)
|
||||
heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
cadvisorInterface := &cadvisortest.Fake{
|
||||
NodeName: config.NodeName,
|
||||
}
|
||||
|
||||
var containerManager cm.ContainerManager
|
||||
if config.ExtendedResources != nil {
|
||||
extendedResources := v1.ResourceList{}
|
||||
for k, v := range config.ExtendedResources {
|
||||
extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
|
||||
}
|
||||
|
||||
containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
|
||||
} else {
|
||||
containerManager = cm.NewStubContainerManager()
|
||||
}
|
||||
|
||||
endpoint, err := fakeremote.GenerateEndpoint()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
|
||||
}
|
||||
fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
|
||||
if err = fakeRemoteRuntime.Start(endpoint); err != nil {
|
||||
return fmt.Errorf("Failed to start fake runtime, error: %w", err)
|
||||
}
|
||||
defer fakeRemoteRuntime.Stop()
|
||||
runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to init runtime service, error: %w", err)
|
||||
}
|
||||
|
||||
var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
|
||||
if config.UseHostImageService {
|
||||
imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to init image service, error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
hollowKubelet := kubemark.NewHollowKubelet(
|
||||
f, c,
|
||||
client,
|
||||
heartbeatClient,
|
||||
cadvisorInterface,
|
||||
imageService,
|
||||
runtimeService,
|
||||
containerManager,
|
||||
)
|
||||
hollowKubelet.Run()
|
||||
}
|
||||
|
||||
if config.Morph == "proxy" {
|
||||
clientConfig.UserAgent = "hollow-proxy"
|
||||
|
||||
client, err := clientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create API Server client, error: %w", err)
|
||||
}
|
||||
iptInterface := fakeiptables.NewFake()
|
||||
sysctl := fakesysctl.NewFake()
|
||||
execer := &fakeexec.FakeExec{
|
||||
LookPathFunc: func(_ string) (string, error) { return "", errors.New("fake execer") },
|
||||
}
|
||||
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
|
||||
recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
|
||||
|
||||
hollowProxy, err := kubemark.NewHollowProxyOrDie(
|
||||
config.NodeName,
|
||||
client,
|
||||
client.CoreV1(),
|
||||
iptInterface,
|
||||
sysctl,
|
||||
execer,
|
||||
eventBroadcaster,
|
||||
recorder,
|
||||
config.UseRealProxier,
|
||||
config.ProxierSyncPeriod,
|
||||
config.ProxierMinSyncPeriod,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create hollowProxy instance, error: %w", err)
|
||||
}
|
||||
return hollowProxy.Run()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
94
cmd/kubemark/app/hollow_node_test.go
Normal file
94
cmd/kubemark/app/hollow_node_test.go
Normal file
@ -0,0 +1,94 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const fakeKubeconfig = `
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: %s
|
||||
name: default
|
||||
contexts:
|
||||
- context:
|
||||
cluster: default
|
||||
user: default
|
||||
name: default
|
||||
current-context: default
|
||||
users:
|
||||
- name: default
|
||||
user:
|
||||
username: config
|
||||
`
|
||||
|
||||
// TestHollowNode is a naive test that attempts to start hollow node and checks if it's not crashing.
|
||||
// Such test is sufficient to detect e.g. missing kubelet dependencies that are not added in
|
||||
// pkg/kubemark/hollow_kubelet.go.
|
||||
func TestHollowNode(t *testing.T) {
|
||||
// temp dir
|
||||
tmpDir, err := os.MkdirTemp("", "hollow-node")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
// https server
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
w.Write([]byte(`ok`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
kubeconfigPath := filepath.Join(tmpDir, "config.kubeconfig")
|
||||
if err := os.WriteFile(kubeconfigPath, []byte(fmt.Sprintf(fakeKubeconfig, server.URL)), os.FileMode(0600)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for morph := range knownMorphs {
|
||||
morph := morph
|
||||
t.Run(morph, func(t *testing.T) {
|
||||
s := &hollowNodeConfig{
|
||||
KubeconfigPath: kubeconfigPath,
|
||||
Morph: morph,
|
||||
}
|
||||
errCh := make(chan error)
|
||||
go func() {
|
||||
data, err := os.ReadFile(kubeconfigPath)
|
||||
t.Logf("read %d, err=%v\n", len(data), err)
|
||||
errCh <- run(s)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
t.Fatalf("Run finished unexpectedly with error: %v", err)
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Logf("Morph %q hasn't crashed for 3s. Calling success.", morph)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -17,294 +17,14 @@ limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
goflag "flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
oteltrace "go.opentelemetry.io/otel/trace"
|
||||
internalapi "k8s.io/cri-api/pkg/apis"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/tools/events"
|
||||
"k8s.io/component-base/cli"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
|
||||
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
|
||||
"k8s.io/component-base/version"
|
||||
"k8s.io/component-base/version/verflag"
|
||||
fakesysctl "k8s.io/component-helpers/node/util/sysctl/testing"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/cluster/ports"
|
||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||
"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cri/remote"
|
||||
fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
|
||||
"k8s.io/kubernetes/pkg/kubemark"
|
||||
utilflag "k8s.io/kubernetes/pkg/util/flag"
|
||||
fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
|
||||
fakeexec "k8s.io/utils/exec/testing"
|
||||
"k8s.io/kubernetes/cmd/kubemark/app"
|
||||
)
|
||||
|
||||
type hollowNodeConfig struct {
|
||||
KubeconfigPath string
|
||||
BootstrapKubeconfigPath string
|
||||
CertDirectory string
|
||||
KubeletPort int
|
||||
KubeletReadOnlyPort int
|
||||
Morph string
|
||||
NodeName string
|
||||
ServerPort int
|
||||
ContentType string
|
||||
UseRealProxier bool
|
||||
ProxierSyncPeriod time.Duration
|
||||
ProxierMinSyncPeriod time.Duration
|
||||
NodeLabels map[string]string
|
||||
RegisterWithTaints []v1.Taint
|
||||
MaxPods int
|
||||
ExtendedResources map[string]string
|
||||
UseHostImageService bool
|
||||
}
|
||||
|
||||
const (
|
||||
maxPods = 110
|
||||
podsPerCore = 0
|
||||
)
|
||||
|
||||
// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
|
||||
// and make the config driven.
|
||||
var knownMorphs = sets.NewString("kubelet", "proxy")
|
||||
|
||||
func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
|
||||
fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
|
||||
fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
|
||||
fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
|
||||
fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
|
||||
fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
|
||||
fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
|
||||
fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
|
||||
fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
|
||||
fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
|
||||
fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
|
||||
fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.")
|
||||
fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.")
|
||||
bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
|
||||
fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
|
||||
fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
|
||||
fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
|
||||
bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
|
||||
fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
|
||||
fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
|
||||
clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
|
||||
}
|
||||
config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
|
||||
}
|
||||
config.ContentType = c.ContentType
|
||||
config.QPS = 10
|
||||
config.Burst = 20
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) bootstrapClientConfig() error {
|
||||
if c.BootstrapKubeconfigPath != "" {
|
||||
return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
|
||||
return &kubemark.HollowKubeletOptions{
|
||||
NodeName: c.NodeName,
|
||||
KubeletPort: c.KubeletPort,
|
||||
KubeletReadOnlyPort: c.KubeletReadOnlyPort,
|
||||
MaxPods: c.MaxPods,
|
||||
PodsPerCore: podsPerCore,
|
||||
NodeLabels: c.NodeLabels,
|
||||
RegisterWithTaints: c.RegisterWithTaints,
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
command := newHollowNodeCommand()
|
||||
command := app.NewHollowNodeCommand()
|
||||
code := cli.Run(command)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// newControllerManagerCommand creates a *cobra.Command object with default parameters
|
||||
func newHollowNodeCommand() *cobra.Command {
|
||||
s := &hollowNodeConfig{
|
||||
NodeLabels: make(map[string]string),
|
||||
ExtendedResources: make(map[string]string),
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "kubemark",
|
||||
Long: "kubemark",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
verflag.PrintAndExitIfRequested()
|
||||
return run(cmd, s)
|
||||
},
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
for _, arg := range args {
|
||||
if len(arg) > 0 {
|
||||
return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
fs := cmd.Flags()
|
||||
fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
|
||||
s.addFlags(fs)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func run(cmd *cobra.Command, config *hollowNodeConfig) error {
|
||||
// To help debugging, immediately log version and print flags.
|
||||
klog.Infof("Version: %+v", version.Get())
|
||||
cliflag.PrintFlags(cmd.Flags())
|
||||
|
||||
if !knownMorphs.Has(config.Morph) {
|
||||
return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
|
||||
}
|
||||
|
||||
// create a client to communicate with API server.
|
||||
err := config.bootstrapClientConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
|
||||
}
|
||||
clientConfig, err := config.createClientConfigFromFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
if config.Morph == "kubelet" {
|
||||
clientConfig.UserAgent = "hollow-kubelet"
|
||||
client, err := clientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
|
||||
|
||||
heartbeatClientConfig := *clientConfig
|
||||
heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
|
||||
// The timeout is the minimum of the lease duration and status update frequency
|
||||
leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
|
||||
if heartbeatClientConfig.Timeout > leaseTimeout {
|
||||
heartbeatClientConfig.Timeout = leaseTimeout
|
||||
}
|
||||
|
||||
heartbeatClientConfig.QPS = float32(-1)
|
||||
heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
|
||||
}
|
||||
|
||||
cadvisorInterface := &cadvisortest.Fake{
|
||||
NodeName: config.NodeName,
|
||||
}
|
||||
|
||||
var containerManager cm.ContainerManager
|
||||
if config.ExtendedResources != nil {
|
||||
extendedResources := v1.ResourceList{}
|
||||
for k, v := range config.ExtendedResources {
|
||||
extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
|
||||
}
|
||||
|
||||
containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
|
||||
} else {
|
||||
containerManager = cm.NewStubContainerManager()
|
||||
}
|
||||
|
||||
endpoint, err := fakeremote.GenerateEndpoint()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
|
||||
}
|
||||
fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
|
||||
if err = fakeRemoteRuntime.Start(endpoint); err != nil {
|
||||
return fmt.Errorf("Failed to start fake runtime, error: %w", err)
|
||||
}
|
||||
defer fakeRemoteRuntime.Stop()
|
||||
runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to init runtime service, error: %w", err)
|
||||
}
|
||||
|
||||
var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
|
||||
if config.UseHostImageService {
|
||||
imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to init image service, error: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
hollowKubelet := kubemark.NewHollowKubelet(
|
||||
f, c,
|
||||
client,
|
||||
heartbeatClient,
|
||||
cadvisorInterface,
|
||||
imageService,
|
||||
runtimeService,
|
||||
containerManager,
|
||||
)
|
||||
hollowKubelet.Run()
|
||||
}
|
||||
|
||||
if config.Morph == "proxy" {
|
||||
clientConfig.UserAgent = "hollow-proxy"
|
||||
|
||||
client, err := clientset.NewForConfig(clientConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create API Server client, error: %w", err)
|
||||
}
|
||||
iptInterface := fakeiptables.NewFake()
|
||||
sysctl := fakesysctl.NewFake()
|
||||
execer := &fakeexec.FakeExec{
|
||||
LookPathFunc: func(_ string) (string, error) { return "", errors.New("fake execer") },
|
||||
}
|
||||
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
|
||||
recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
|
||||
|
||||
hollowProxy, err := kubemark.NewHollowProxyOrDie(
|
||||
config.NodeName,
|
||||
client,
|
||||
client.CoreV1(),
|
||||
iptInterface,
|
||||
sysctl,
|
||||
execer,
|
||||
eventBroadcaster,
|
||||
recorder,
|
||||
config.UseRealProxier,
|
||||
config.ProxierSyncPeriod,
|
||||
config.ProxierMinSyncPeriod,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create hollowProxy instance, error: %w", err)
|
||||
}
|
||||
return hollowProxy.Run()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -20,12 +20,14 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/mount-utils"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/record"
|
||||
internalapi "k8s.io/cri-api/pkg/apis"
|
||||
kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
|
||||
"k8s.io/kubernetes/cmd/kubelet/app/options"
|
||||
@ -111,6 +113,8 @@ func NewHollowKubelet(
|
||||
Subpather: &subpath.FakeSubpath{},
|
||||
HostUtil: hostutil.NewFakeHostUtil(nil),
|
||||
PodStartupLatencyTracker: kubeletutil.NewPodStartupLatencyTracker(),
|
||||
TracerProvider: trace.NewNoopTracerProvider(),
|
||||
Recorder: &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
|
||||
}
|
||||
|
||||
return &HollowKubelet{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user