From ca335d7be9f4507ff509b4f838e1d337b776ec55 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sat, 11 Apr 2015 12:45:45 -0400 Subject: [PATCH] Add support for chaos to Kubelet and hack/local-up-cluster.sh --- cmd/kubelet/app/server.go | 29 +++++++++++++++++++++++++++-- hack/local-up-cluster.sh | 2 ++ pkg/kubectl/cmd/update.go | 1 - 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index d5528adece3..79dd3c8ab57 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -30,6 +30,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/chaosclient" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" "github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth" "github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider" @@ -84,7 +85,6 @@ type KubeletServer struct { ClusterDomain string MasterServiceNamespace string ClusterDNS util.IP - ReallyCrashForTesting bool StreamingConnectionIdleTimeout time.Duration ImageGCHighThresholdPercent int ImageGCLowThresholdPercent int @@ -95,6 +95,13 @@ type KubeletServer struct { TLSPrivateKeyFile string CertDirectory string NodeStatusUpdateFrequency time.Duration + + // Flags intended for testing + + // Crash immediately, rather than eating panics. + ReallyCrashForTesting bool + // Insert a probability of random errors during calls to the master. + ChaosChance float64 } // bootstrapping interface for kubelet, targets the initialization protocol @@ -181,7 +188,6 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.ClusterDomain, "cluster_domain", s.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains") fs.StringVar(&s.MasterServiceNamespace, "master_service_namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") fs.Var(&s.ClusterDNS, "cluster_dns", "IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers") - fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, crash with panics more often.") fs.DurationVar(&s.StreamingConnectionIdleTimeout, "streaming_connection_idle_timeout", 0, "Maximum time a streaming connection can be idle before the connection is automatically closed. Example: '5m'") fs.DurationVar(&s.NodeStatusUpdateFrequency, "node_status_update_frequency", s.NodeStatusUpdateFrequency, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s") fs.IntVar(&s.ImageGCHighThresholdPercent, "image_gc_high_threshold", s.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Default: 90%%") @@ -189,6 +195,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.NetworkPluginName, "network_plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + + // Flags intended for testing, not recommended used in production environments. + fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") + fs.Float64Var(&s.ChaosChance, "chaos_chance", s.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]") } // Run runs the specified KubeletServer. This should never exit. @@ -337,6 +347,9 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) { glog.Infof("Multiple api servers specified. Picking first one") } clientConfig.Host = s.APIServerList[0] + + s.addChaosToClientConfig(&clientConfig) + c, err := client.New(&clientConfig) if err != nil { return nil, err @@ -344,6 +357,18 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) { return c, nil } +// addChaosToClientConfig injects random errors into client connections if configured. +func (s *KubeletServer) addChaosToClientConfig(config *client.Config) { + if s.ChaosChance != 0.0 { + config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { + seed := chaosclient.NewSeed(1) + // TODO: introduce a standard chaos package with more tunables - this is just a proof of concept + // TODO: introduce random latency and stalls + return chaosclient.NewChaosRoundTripper(rt, chaosclient.LogChaos, seed.P(s.ChaosChance, chaosclient.ErrSimulatedConnectionResetByPeer)) + } + } +} + // SimpleRunKubelet is a simple way to start a Kubelet talking to dockerEndpoint, using an API Client. // Under the hood it calls RunKubelet (below) func SimpleKubelet(client *client.Client, diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 08c8a0d06d1..531f257c516 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -45,6 +45,7 @@ API_HOST=${API_HOST:-127.0.0.1} API_CORS_ALLOWED_ORIGINS=${API_CORS_ALLOWED_ORIGINS:-"/127.0.0.1(:[0-9]+)?$,/localhost(:[0-9]+)?$"} KUBELET_PORT=${KUBELET_PORT:-10250} LOG_LEVEL=${LOG_LEVEL:-3} +CHAOS_CHANCE=${CHAOS_CHANCE:-0.0} # For the common local scenario, fail fast if server is already running. # this can happen if you run local-up-cluster.sh twice and kill etcd in between. @@ -139,6 +140,7 @@ CTLRMGR_PID=$! KUBELET_LOG=/tmp/kubelet.log sudo -E "${GO_OUT}/kubelet" \ --v=${LOG_LEVEL} \ + --chaos_chance="${CHAOS_CHANCE}" \ --hostname_override="127.0.0.1" \ --address="127.0.0.1" \ --api_servers="${API_HOST}:${API_PORT}" \ diff --git a/pkg/kubectl/cmd/update.go b/pkg/kubectl/cmd/update.go index cbca1e6f3c0..d8d3323d03a 100644 --- a/pkg/kubectl/cmd/update.go +++ b/pkg/kubectl/cmd/update.go @@ -114,7 +114,6 @@ func RunUpdate(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []str fmt.Fprintf(out, "%s/%s\n", info.Mapping.Resource, info.Name) return nil }) - } func updateWithPatch(cmd *cobra.Command, args []string, f *cmdutil.Factory, patch string) (string, error) {