git: synchronize update-index with a mutex (#4130)

if `pkglib.NewFromConfig` is used in parallel, it calls
```
git -C /some/directory update-index -q --refresh
```
in parallel.

But `git` does not like this and exits with 128.

This can be easily tried with:
```
git -C /some/dir update-index -q --refresh & \
git -C /some/dir update-index -q --refresh
```

Signed-off-by: Christoph Ostarek <christoph@zededa.com>

src/cmd/linuxkit/pkglib/docker.go

@@ -276,6 +276,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 	)
 	for range buildKitCheckRetryCount {
 		var b bytes.Buffer
+		var cid string
 		if err := dr.command(nil, &b, io.Discard, "--context", dockerContext, "container", "inspect", name); err == nil {
 			// we already have a container named "linuxkit-builder" in the provided context.
 			// get its state and config
@@ -284,6 +285,7 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 				return nil, fmt.Errorf("unable to read results of 'container inspect %s': %v", name, err)
 			}
 
+			cid = containerJSON[0].ID
 			existingImage := containerJSON[0].Config.Image
 			isRunning := containerJSON[0].State.Status == "running"
 
@@ -326,13 +328,25 @@ func (dr *dockerRunnerImpl) builderEnsureContainer(ctx context.Context, name, im
 		// if we made it here, we need to stop and remove the container, either because of a config mismatch,
 		// or because we received the CLI option
 		if stop {
-			if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", name); err != nil {
-				return nil, fmt.Errorf("failed to stop existing container %s", name)
+			if cid == "" {
+				// we don't have a container ID, so we can't stop it
+				return nil, fmt.Errorf("unable to stop existing container %s, no ID found", name)
+			}
+			if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "stop", cid); err != nil {
+				// if we failed, do a retry; maybe it does not even exist anymore
+				time.Sleep(buildkitCheckInterval)
+				continue
 			}
 		}
 		if remove {
-			if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", name); err != nil {
-				return nil, fmt.Errorf("failed to remove existing container %s", name)
+			if cid == "" {
+				// we don't have a container ID, so we can't remove it
+				return nil, fmt.Errorf("unable to remove existing container %s, no ID found", name)
+			}
+			if err := dr.command(nil, io.Discard, io.Discard, "--context", dockerContext, "container", "rm", cid); err != nil {
+				// if we failed, do a retry; maybe it does not even exist anymore
+				time.Sleep(buildkitCheckInterval)
+				continue
 			}
 		}
 		if recreate {

src/cmd/linuxkit/pkglib/git.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -27,6 +28,8 @@ type git struct {
 	dir string
 }
 
+var gitMutex sync.Mutex
+
 // Returns git==nil and no error if the path is not within a git repository
 func newGit(dir string) (*git, error) {
 	g := &git{dir}
@@ -190,9 +193,12 @@ func (g git) isDirty(pkg, commit string) (bool, error) {
 	// because `git diff-index` only uses the `lstat` result and
 	// not the actual file contents. Running `git update-index
 	// --refresh` updates the cache.
+	gitMutex.Lock()
 	if err := g.command("update-index", "-q", "--refresh"); err != nil {
+		gitMutex.Unlock()
 		return false, err
 	}
+	gitMutex.Unlock()
 
 	// diff-index works pretty well, except that
 	err := g.command("diff-index", "--quiet", commit, "--", pkg)