From d16eee0fee2edb635c03c4f0e4c6bd7a5e2ea591 Mon Sep 17 00:00:00 2001 From: Benjamin Pineau Date: Tue, 18 Aug 2020 13:07:18 +0200 Subject: [PATCH] ARM client: survive empty response and error We're seeing legacy-cloud-providersazure/clients (in our case, the synchronized copy used by cluster-autoscaler) segfaulting under heavy pressure and ARM throttling, like so: ``` panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x10 pc=0x1b595b5] cluster-autoscaler-all-79b9478bf5-cgkg8 cluster-autoscaler goroutine 82 [running]: k8s.io/autoscaler/cluster-autoscaler/vendor/k8s.io/legacy-cloud-providers/azure/clients/armclient.(*Client).Send(0xc00052f520, 0x3b6dc40, 0xc000937440, 0xc000933f00, 0x0, 0x0) /home/jb/go/src/k8s.io/autoscaler/cluster-autoscaler/vendor/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go:122 +0xb5 ``` Reason is the ARM client expects `sendRequest()` to return either a non nil *retry.Error, or a non nil *http.Response, while both can be nil. --- .../azure/clients/armclient/azure_armclient.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go index c595e9d068a..0854c8e8fcf 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/clients/armclient/azure_armclient.go @@ -109,6 +109,11 @@ func (c *Client) sendRequest(ctx context.Context, request *http.Request) (*http. request, retry.DoExponentialBackoffRetry(&sendBackoff), ) + + if response == nil && err == nil { + return response, retry.NewError(false, fmt.Errorf("Empty response and no HTTP code")) + } + return response, retry.GetError(response, err) }