pkg build: handle race condition where builder is started at same time (#4127)

Signed-off-by: Avi Deitcher <avi@deitcher.net>

src/cmd/linuxkit/pkglib/docker.go

@@ -263,9 +263,19 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 		// stop existing one
 		stop   = false
 		remove = false
-		b      bytes.Buffer
+		found  = false
 	)
 
+	const (
+
+		// we will retry starting the container 3 times, waiting 1 second between each retry
+		// this is to allow for race conditions, where we inspected, didn't find it,
+		// some other process created it, and we are now trying to create it.
+		buildkitCheckInterval   = 1 * time.Second
+		buildKitCheckRetryCount = 3
+	)
+	for range buildKitCheckRetryCount {
+		var b bytes.Buffer
 		if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
 			// we already have a container named "linuxkit-builder" in the provided context.
 			// get its state and config
@@ -302,6 +312,8 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 				return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext))
 			default:
 				// we have an existing container, but it isn't running, so start it
+				// note that if it somehow got started in a parallel process or thread,
+				// `container start` is a no-op, so we will get no errors; this just works.
 				fmt.Printf("starting existing container %s\n", name)
 				if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil {
 					return nil, fmt.Errorf("failed to start existing container %s", name)
@@ -328,14 +340,23 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 			args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"}
 			msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext)
 			fmt.Println(msg)
-		if err := dr.command(nil, nil, nil, args...); err != nil {
+			if err := dr.command(nil, nil, io.Discard, args...); err != nil {
-			return nil, err
+				// if we failed, do a retry
+				time.Sleep(buildkitCheckInterval)
+				continue
 			}
 		}
+		found = true
+		break
+	}
+	if !found {
+		return nil, fmt.Errorf("unable to create or find builder container %s in context %s after %d retries", name, dockerContext, buildKitCheckRetryCount)
+	}
+
 	// wait for buildkit socket to be ready up to the timeout
 	fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer)
 	timeout := time.After(buildkitWaitServer * time.Second)
-	ticker := time.NewTicker(buildkitCheckInterval * time.Second)
+	ticker := time.NewTicker(buildkitCheckInterval)
 	// Keep trying until we're timed out or get a success
 	for {
 		select {