github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-23 19:05:37 +00:00
Code Issues Projects Releases Wiki Activity

pkg build: handle race condition where builder is started at same time

Browse Source 
Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
Avi Deitcher 2025-06-24 22:18:06 +03:00
parent b42e1a8bab
commit ad6dd42a63
1 changed files with 82 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
src/cmd/linuxkit/pkglib/docker.go
View File

@ -263,79 +263,100 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
// stop existing one // stop existing one
stop = false stop = false
remove = false remove = false
b bytes.Buffer found = false
) )
if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil { const (
// we already have a container named "linuxkit-builder" in the provided context.
// get its state and config
var containerJSON []dockercontainertypes.InspectResponse
if err := json.Unmarshal(b.Bytes(), &containerJSON); err != nil || len(containerJSON) < 1 {
return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
}
existingImage := containerJSON[0].Config.Image // we will retry starting the container 3 times, waiting 1 second between each retry
isRunning := containerJSON[0].State.Status == "running" // this is to allow for race conditions, where we inspected, didn't find it,
// some other process created it, and we are now trying to create it.
switch { buildkitCheckInterval = 1 * time.Second
case forceRestart: buildKitCheckRetryCount = 3
// if forceRestart==true, we always recreate, else we check if it matches our requirements )
fmt.Printf("told to force restart, replacing existing container %s\n", name) for range buildKitCheckRetryCount {
recreate = true var b bytes.Buffer
stop = isRunning if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
remove = true // we already have a container named "linuxkit-builder" in the provided context.
case existingImage != image: // get its state and config
// if image mismatches, recreate var containerJSON []dockercontainertypes.InspectResponse
fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image) if err := json.Unmarshal(b.Bytes(), &containerJSON); err != nil || len(containerJSON) < 1 {
recreate = true return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
stop = isRunning }
remove = true
case !containerJSON[0].HostConfig.Privileged: existingImage := containerJSON[0].Config.Image
// if unprivileged, we need to remove it and start a new container with the right permissions isRunning := containerJSON[0].State.Status == "running"
fmt.Printf("existing container %s is unprivileged, replacing\n", name)
recreate = true switch {
stop = isRunning case forceRestart:
remove = true // if forceRestart==true, we always recreate, else we check if it matches our requirements
case isRunning: fmt.Printf("told to force restart, replacing existing container %s\n", name)
// if already running with the right image and permissions, just use it recreate = true
fmt.Printf("using existing container %s\n", name) stop = isRunning
return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext)) remove = true
default: case existingImage != image:
// we have an existing container, but it isn't running, so start it // if image mismatches, recreate
fmt.Printf("starting existing container %s\n", name) fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image)
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil { recreate = true
return nil, fmt.Errorf("failed to start existing container %s", name) stop = isRunning
remove = true
case !containerJSON[0].HostConfig.Privileged:
// if unprivileged, we need to remove it and start a new container with the right permissions
fmt.Printf("existing container %s is unprivileged, replacing\n", name)
recreate = true
stop = isRunning
remove = true
case isRunning:
// if already running with the right image and permissions, just use it
fmt.Printf("using existing container %s\n", name)
return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext))
default:
// we have an existing container, but it isn't running, so start it
// note that if it somehow got started in a parallel process or thread,
// `container start` is a no-op, so we will get no errors; this just works.
fmt.Printf("starting existing container %s\n", name)
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil {
return nil, fmt.Errorf("failed to start existing container %s", name)
}
recreate = false
stop = false
remove = false
} }
recreate = false
stop = false
remove = false
} }
} // if we made it here, we need to stop and remove the container, either because of a config mismatch,
// if we made it here, we need to stop and remove the container, either because of a config mismatch, // or because we received the CLI option
// or because we received the CLI option if stop {
if stop { if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil {
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil { return nil, fmt.Errorf("failed to stop existing container %s", name)
return nil, fmt.Errorf("failed to stop existing container %s", name) }
} }
} if remove {
if remove { if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil {
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil { return nil, fmt.Errorf("failed to remove existing container %s", name)
return nil, fmt.Errorf("failed to remove existing container %s", name) }
} }
} if recreate {
if recreate { // create the builder
// create the builder args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"}
args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"} msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext)
msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext) fmt.Println(msg)
fmt.Println(msg) if err := dr.command(nil, nil, io.Discard, args...); err != nil {
if err := dr.command(nil, nil, nil, args...); err != nil { // if we failed, do a retry
return nil, err time.Sleep(buildkitCheckInterval)
continue
}
} }
found = true
break
} }
if !found {
return nil, fmt.Errorf("unable to create or find builder container %s in context %s after %d retries", name, dockerContext, buildKitCheckRetryCount)
}
// wait for buildkit socket to be ready up to the timeout // wait for buildkit socket to be ready up to the timeout
fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer) fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer)
timeout := time.After(buildkitWaitServer * time.Second) timeout := time.After(buildkitWaitServer * time.Second)
ticker := time.NewTicker(buildkitCheckInterval * time.Second) ticker := time.NewTicker(buildkitCheckInterval)
// Keep trying until we're timed out or get a success // Keep trying until we're timed out or get a success
for { for {
select { select {