github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-13 13:14:05 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #6729 from smarterclayton/chaosclient

Browse Source 
Add a new Chaos transport that can simulate network failure and add it to the kubelet
This commit is contained in:
Fabio Yeon
2015-04-13 16:06:53 -07:00
parent da481aacd7 ca335d7be9
commit e99141de0d
6 changed files with 269 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/kubelet/app/server.go
View File

@@ -31,6 +31,7 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/chaosclient"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
"github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider"
@@ -85,7 +86,6 @@ type KubeletServer struct {
ClusterDomain string
MasterServiceNamespace string
ClusterDNS util.IP
ReallyCrashForTesting bool
StreamingConnectionIdleTimeout time.Duration
ImageGCHighThresholdPercent int
ImageGCLowThresholdPercent int
@@ -96,6 +96,13 @@ type KubeletServer struct {
TLSPrivateKeyFile string
CertDirectory string
NodeStatusUpdateFrequency time.Duration
// Flags intended for testing
// Crash immediately, rather than eating panics.
ReallyCrashForTesting bool
// Insert a probability of random errors during calls to the master.
ChaosChance float64
}
// bootstrapping interface for kubelet, targets the initialization protocol
@@ -182,7 +189,6 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.ClusterDomain, "cluster_domain", s.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
fs.StringVar(&s.MasterServiceNamespace, "master_service_namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods")
fs.Var(&s.ClusterDNS, "cluster_dns", "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, crash with panics more often.")
fs.DurationVar(&s.StreamingConnectionIdleTimeout, "streaming_connection_idle_timeout", 0, "Maximum time a streaming connection can be idle before the connection is automatically closed. Example: '5m'")
fs.DurationVar(&s.NodeStatusUpdateFrequency, "node_status_update_frequency", s.NodeStatusUpdateFrequency, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s")
fs.IntVar(&s.ImageGCHighThresholdPercent, "image_gc_high_threshold", s.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Default: 90%%")
@@ -190,6 +196,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.NetworkPluginName, "network_plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.")
fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
// Flags intended for testing, not recommended used in production environments.
fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")
fs.Float64Var(&s.ChaosChance, "chaos_chance", s.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]")
}
// Run runs the specified KubeletServer. This should never exit.
@@ -338,6 +348,9 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) {
glog.Infof("Multiple api servers specified. Picking first one")
}
clientConfig.Host = s.APIServerList[0]
s.addChaosToClientConfig(&clientConfig)
c, err := client.New(&clientConfig)
if err != nil {
return nil, err
@@ -345,6 +358,18 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) {
return c, nil
}
// addChaosToClientConfig injects random errors into client connections if configured.
func (s *KubeletServer) addChaosToClientConfig(config *client.Config) {
if s.ChaosChance != 0.0 {
config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
seed := chaosclient.NewSeed(1)
// TODO: introduce a standard chaos package with more tunables - this is just a proof of concept
// TODO: introduce random latency and stalls
return chaosclient.NewChaosRoundTripper(rt, chaosclient.LogChaos, seed.P(s.ChaosChance, chaosclient.ErrSimulatedConnectionResetByPeer))
}
}
}
// SimpleRunKubelet is a simple way to start a Kubelet talking to dockerEndpoint, using an API Client.
// Under the hood it calls RunKubelet (below)
func SimpleKubelet(client *client.Client,