mirror of
https://github.com/linuxkit/linuxkit.git
synced 2025-07-24 19:28:09 +00:00
pkg build: handle race condition where builder is started at same time
Signed-off-by: Avi Deitcher <avi@deitcher.net>
This commit is contained in:
parent
b42e1a8bab
commit
ad6dd42a63
@ -263,79 +263,100 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
|
|||||||
// stop existing one
|
// stop existing one
|
||||||
stop = false
|
stop = false
|
||||||
remove = false
|
remove = false
|
||||||
b bytes.Buffer
|
found = false
|
||||||
)
|
)
|
||||||
|
|
||||||
if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
|
const (
|
||||||
// we already have a container named "linuxkit-builder" in the provided context.
|
|
||||||
// get its state and config
|
|
||||||
var containerJSON []dockercontainertypes.InspectResponse
|
|
||||||
if err := json.Unmarshal(b.Bytes(), &containerJSON); err != nil || len(containerJSON) < 1 {
|
|
||||||
return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
existingImage := containerJSON[0].Config.Image
|
// we will retry starting the container 3 times, waiting 1 second between each retry
|
||||||
isRunning := containerJSON[0].State.Status == "running"
|
// this is to allow for race conditions, where we inspected, didn't find it,
|
||||||
|
// some other process created it, and we are now trying to create it.
|
||||||
switch {
|
buildkitCheckInterval = 1 * time.Second
|
||||||
case forceRestart:
|
buildKitCheckRetryCount = 3
|
||||||
// if forceRestart==true, we always recreate, else we check if it matches our requirements
|
)
|
||||||
fmt.Printf("told to force restart, replacing existing container %s\n", name)
|
for range buildKitCheckRetryCount {
|
||||||
recreate = true
|
var b bytes.Buffer
|
||||||
stop = isRunning
|
if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
|
||||||
remove = true
|
// we already have a container named "linuxkit-builder" in the provided context.
|
||||||
case existingImage != image:
|
// get its state and config
|
||||||
// if image mismatches, recreate
|
var containerJSON []dockercontainertypes.InspectResponse
|
||||||
fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image)
|
if err := json.Unmarshal(b.Bytes(), &containerJSON); err != nil || len(containerJSON) < 1 {
|
||||||
recreate = true
|
return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
|
||||||
stop = isRunning
|
}
|
||||||
remove = true
|
|
||||||
case !containerJSON[0].HostConfig.Privileged:
|
existingImage := containerJSON[0].Config.Image
|
||||||
// if unprivileged, we need to remove it and start a new container with the right permissions
|
isRunning := containerJSON[0].State.Status == "running"
|
||||||
fmt.Printf("existing container %s is unprivileged, replacing\n", name)
|
|
||||||
recreate = true
|
switch {
|
||||||
stop = isRunning
|
case forceRestart:
|
||||||
remove = true
|
// if forceRestart==true, we always recreate, else we check if it matches our requirements
|
||||||
case isRunning:
|
fmt.Printf("told to force restart, replacing existing container %s\n", name)
|
||||||
// if already running with the right image and permissions, just use it
|
recreate = true
|
||||||
fmt.Printf("using existing container %s\n", name)
|
stop = isRunning
|
||||||
return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext))
|
remove = true
|
||||||
default:
|
case existingImage != image:
|
||||||
// we have an existing container, but it isn't running, so start it
|
// if image mismatches, recreate
|
||||||
fmt.Printf("starting existing container %s\n", name)
|
fmt.Printf("existing container %s is running image %s instead of target %s, replacing\n", name, existingImage, image)
|
||||||
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil {
|
recreate = true
|
||||||
return nil, fmt.Errorf("failed to start existing container %s", name)
|
stop = isRunning
|
||||||
|
remove = true
|
||||||
|
case !containerJSON[0].HostConfig.Privileged:
|
||||||
|
// if unprivileged, we need to remove it and start a new container with the right permissions
|
||||||
|
fmt.Printf("existing container %s is unprivileged, replacing\n", name)
|
||||||
|
recreate = true
|
||||||
|
stop = isRunning
|
||||||
|
remove = true
|
||||||
|
case isRunning:
|
||||||
|
// if already running with the right image and permissions, just use it
|
||||||
|
fmt.Printf("using existing container %s\n", name)
|
||||||
|
return buildkitClient.New(ctx, fmt.Sprintf("docker-container://%s?context=%s", name, dockerContext))
|
||||||
|
default:
|
||||||
|
// we have an existing container, but it isn't running, so start it
|
||||||
|
// note that if it somehow got started in a parallel process or thread,
|
||||||
|
// `container start` is a no-op, so we will get no errors; this just works.
|
||||||
|
fmt.Printf("starting existing container %s\n", name)
|
||||||
|
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "start", name); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to start existing container %s", name)
|
||||||
|
}
|
||||||
|
recreate = false
|
||||||
|
stop = false
|
||||||
|
remove = false
|
||||||
}
|
}
|
||||||
recreate = false
|
|
||||||
stop = false
|
|
||||||
remove = false
|
|
||||||
}
|
}
|
||||||
}
|
// if we made it here, we need to stop and remove the container, either because of a config mismatch,
|
||||||
// if we made it here, we need to stop and remove the container, either because of a config mismatch,
|
// or because we received the CLI option
|
||||||
// or because we received the CLI option
|
if stop {
|
||||||
if stop {
|
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil {
|
||||||
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil {
|
return nil, fmt.Errorf("failed to stop existing container %s", name)
|
||||||
return nil, fmt.Errorf("failed to stop existing container %s", name)
|
}
|
||||||
}
|
}
|
||||||
}
|
if remove {
|
||||||
if remove {
|
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil {
|
||||||
if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil {
|
return nil, fmt.Errorf("failed to remove existing container %s", name)
|
||||||
return nil, fmt.Errorf("failed to remove existing container %s", name)
|
}
|
||||||
}
|
}
|
||||||
}
|
if recreate {
|
||||||
if recreate {
|
// create the builder
|
||||||
// create the builder
|
args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"}
|
||||||
args := []string{"--context", dockerContext, "container", "run", "-d", "--name", name, "--privileged", image, "--allow-insecure-entitlement", "network.host", "--addr", fmt.Sprintf("unix://%s", buildkitSocketPath), "--debug"}
|
msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext)
|
||||||
msg := fmt.Sprintf("creating builder container '%s' in context '%s'", name, dockerContext)
|
fmt.Println(msg)
|
||||||
fmt.Println(msg)
|
if err := dr.command(nil, nil, io.Discard, args...); err != nil {
|
||||||
if err := dr.command(nil, nil, nil, args...); err != nil {
|
// if we failed, do a retry
|
||||||
return nil, err
|
time.Sleep(buildkitCheckInterval)
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
found = true
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
if !found {
|
||||||
|
return nil, fmt.Errorf("unable to create or find builder container %s in context %s after %d retries", name, dockerContext, buildKitCheckRetryCount)
|
||||||
|
}
|
||||||
|
|
||||||
// wait for buildkit socket to be ready up to the timeout
|
// wait for buildkit socket to be ready up to the timeout
|
||||||
fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer)
|
fmt.Printf("waiting for buildkit builder to be ready, up to %d seconds\n", buildkitWaitServer)
|
||||||
timeout := time.After(buildkitWaitServer * time.Second)
|
timeout := time.After(buildkitWaitServer * time.Second)
|
||||||
ticker := time.NewTicker(buildkitCheckInterval * time.Second)
|
ticker := time.NewTicker(buildkitCheckInterval)
|
||||||
// Keep trying until we're timed out or get a success
|
// Keep trying until we're timed out or get a success
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
Loading…
Reference in New Issue
Block a user