bump github.com/moby/buildkit to v0.13.0 (#351)

* bump github.com/moby/buildkit to v0.13.0

Signed-off-by: Nianyu Shen <nianyu@spectrocloud.com>

* fix: update dep usage based on newer version

Signed-off-by: Nianyu Shen <nianyu@spectrocloud.com>

* remove empty line

Signed-off-by: Nianyu Shen <nianyu@spectrocloud.com>

* ci: bump golang to 1.21.x

* Bump moby

* debug

---------

Signed-off-by: Nianyu Shen <nianyu@spectrocloud.com>
Co-authored-by: Nianyu Shen <nianyu@spectrocloud.com>

vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go

@@ -16,7 +16,6 @@ package layout
 
 import (
 	"io"
-	"io/ioutil"
 	"os"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -30,7 +29,7 @@ func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) {
 // Bytes is a convenience function to return a blob from the Path as
 // a byte slice.
 func (l Path) Bytes(h v1.Hash) ([]byte, error) {
-	return ioutil.ReadFile(l.blobPath(h))
+	return os.ReadFile(l.blobPath(h))
 }
 
 func (l Path) blobPath(h v1.Hash) string {

vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go

@@ -19,7 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"os"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
@@ -45,7 +45,7 @@ func ImageIndexFromPath(path string) (v1.ImageIndex, error) {
 
 // ImageIndex returns a v1.ImageIndex for the Path.
 func (l Path) ImageIndex() (v1.ImageIndex, error) {
-	rawIndex, err := ioutil.ReadFile(l.path("index.json"))
+	rawIndex, err := os.ReadFile(l.path("index.json"))
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package layout
 
 import v1 "github.com/google/go-containerregistry/pkg/v1"

vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go

@@ -17,15 +17,18 @@ package layout
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	"github.com/google/go-containerregistry/pkg/logs"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
+	"github.com/google/go-containerregistry/pkg/v1/stream"
 	"github.com/google/go-containerregistry/pkg/v1/types"
 	"golang.org/x/sync/errgroup"
 )
@@ -41,33 +44,17 @@ func (l Path) AppendImage(img v1.Image, options ...Option) error {
 		return err
 	}
 
-	mt, err := img.MediaType()
+	desc, err := partial.Descriptor(img)
 	if err != nil {
 		return err
 	}
 
-	d, err := img.Digest()
-	if err != nil {
-		return err
-	}
-
-	manifest, err := img.RawManifest()
-	if err != nil {
-		return err
-	}
-
-	desc := v1.Descriptor{
-		MediaType: mt,
-		Size:      int64(len(manifest)),
-		Digest:    d,
-	}
-
 	o := makeOptions(options...)
 	for _, opt := range o.descOpts {
-		opt(&desc)
+		opt(desc)
 	}
 
-	return l.AppendDescriptor(desc)
+	return l.AppendDescriptor(*desc)
 }
 
 // AppendIndex writes a v1.ImageIndex to the Path and updates
@@ -77,33 +64,17 @@ func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error {
 		return err
 	}
 
-	mt, err := ii.MediaType()
+	desc, err := partial.Descriptor(ii)
 	if err != nil {
 		return err
 	}
 
-	d, err := ii.Digest()
-	if err != nil {
-		return err
-	}
-
-	manifest, err := ii.RawManifest()
-	if err != nil {
-		return err
-	}
-
-	desc := v1.Descriptor{
-		MediaType: mt,
-		Size:      int64(len(manifest)),
-		Digest:    d,
-	}
-
 	o := makeOptions(options...)
 	for _, opt := range o.descOpts {
-		opt(&desc)
+		opt(desc)
 	}
 
-	return l.AppendDescriptor(desc)
+	return l.AppendDescriptor(*desc)
 }
 
 // AppendDescriptor adds a descriptor to the index.json of the Path.
@@ -215,40 +186,110 @@ func (l Path) WriteFile(name string, data []byte, perm os.FileMode) error {
 		return err
 	}
 
-	return ioutil.WriteFile(l.path(name), data, perm)
+	return os.WriteFile(l.path(name), data, perm)
 }
 
 // WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at
 // blobs/{hash.Algorithm}/{hash.Hex}.
 func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error {
+	return l.writeBlob(hash, -1, r, nil)
+}
+
+func (l Path) writeBlob(hash v1.Hash, size int64, rc io.ReadCloser, renamer func() (v1.Hash, error)) error {
+	if hash.Hex == "" && renamer == nil {
+		panic("writeBlob called an invalid hash and no renamer")
+	}
+
 	dir := l.path("blobs", hash.Algorithm)
 	if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
 		return err
 	}
 
+	// Check if blob already exists and is the correct size
 	file := filepath.Join(dir, hash.Hex)
-	if _, err := os.Stat(file); err == nil {
-		// Blob already exists, that's fine.
+	if s, err := os.Stat(file); err == nil && !s.IsDir() && (s.Size() == size || size == -1) {
 		return nil
 	}
-	w, err := os.Create(file)
+
+	// If a renamer func was provided write to a temporary file
+	open := func() (*os.File, error) { return os.Create(file) }
+	if renamer != nil {
+		open = func() (*os.File, error) { return os.CreateTemp(dir, hash.Hex) }
+	}
+	w, err := open()
 	if err != nil {
 		return err
 	}
+	if renamer != nil {
+		// Delete temp file if an error is encountered before renaming
+		defer func() {
+			if err := os.Remove(w.Name()); err != nil && !errors.Is(err, os.ErrNotExist) {
+				logs.Warn.Printf("error removing temporary file after encountering an error while writing blob: %v", err)
+			}
+		}()
+	}
 	defer w.Close()
 
-	_, err = io.Copy(w, r)
-	return err
+	// Write to file and exit if not renaming
+	if n, err := io.Copy(w, rc); err != nil || renamer == nil {
+		return err
+	} else if size != -1 && n != size {
+		return fmt.Errorf("expected blob size %d, but only wrote %d", size, n)
+	}
+
+	// Always close reader before renaming, since Close computes the digest in
+	// the case of streaming layers. If Close is not called explicitly, it will
+	// occur in a goroutine that is not guaranteed to succeed before renamer is
+	// called. When renamer is the layer's Digest method, it can return
+	// ErrNotComputed.
+	if err := rc.Close(); err != nil {
+		return err
+	}
+
+	// Always close file before renaming
+	if err := w.Close(); err != nil {
+		return err
+	}
+
+	// Rename file based on the final hash
+	finalHash, err := renamer()
+	if err != nil {
+		return fmt.Errorf("error getting final digest of layer: %w", err)
+	}
+
+	renamePath := l.path("blobs", finalHash.Algorithm, finalHash.Hex)
+	return os.Rename(w.Name(), renamePath)
 }
 
-// TODO: A streaming version of WriteBlob so we don't have to know the hash
-// before we write it.
-
-// TODO: For streaming layers we should write to a tmp file then Rename to the
-// final digest.
+// writeLayer writes the compressed layer to a blob. Unlike WriteBlob it will
+// write to a temporary file (suffixed with .tmp) within the layout until the
+// compressed reader is fully consumed and written to disk. Also unlike
+// WriteBlob, it will not skip writing and exit without error when a blob file
+// exists, but does not have the correct size. (The blob hash is not
+// considered, because it may be expensive to compute.)
 func (l Path) writeLayer(layer v1.Layer) error {
 	d, err := layer.Digest()
-	if err != nil {
+	if errors.Is(err, stream.ErrNotComputed) {
+		// Allow digest errors, since streams may not have calculated the hash
+		// yet. Instead, use an empty value, which will be transformed into a
+		// random file name with `os.CreateTemp` and the final digest will be
+		// calculated after writing to a temp file and before renaming to the
+		// final path.
+		d = v1.Hash{Algorithm: "sha256", Hex: ""}
+	} else if err != nil {
+		return err
+	}
+
+	s, err := layer.Size()
+	if errors.Is(err, stream.ErrNotComputed) {
+		// Allow size errors, since streams may not have calculated the size
+		// yet. Instead, use zero as a sentinel value meaning that no size
+		// comparison can be done and any sized blob file should be considered
+		// valid and not overwritten.
+		//
+		// TODO: Provide an option to always overwrite blobs.
+		s = -1
+	} else if err != nil {
 		return err
 	}
 
@@ -257,7 +298,10 @@ func (l Path) writeLayer(layer v1.Layer) error {
 		return err
 	}
 
-	return l.WriteBlob(d, r)
+	if err := l.writeBlob(d, s, r, layer.Digest); err != nil {
+		return fmt.Errorf("error writing layer: %w", err)
+	}
+	return nil
 }
 
 // RemoveBlob removes a file from the blobs directory in the Path
@@ -306,7 +350,7 @@ func (l Path) WriteImage(img v1.Image) error {
 	if err != nil {
 		return err
 	}
-	if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
+	if err := l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
 		return err
 	}
 
@@ -320,7 +364,7 @@ func (l Path) WriteImage(img v1.Image) error {
 		return err
 	}
 
-	return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest)))
+	return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest)))
 }
 
 type withLayer interface {
@@ -415,12 +459,15 @@ func (l Path) WriteIndex(ii v1.ImageIndex) error {
 //
 // The contents are written in the following format:
 // At the top level, there is:
-//   One oci-layout file containing the version of this image-layout.
-//   One index.json file listing descriptors for the contained images.
+//
+//	One oci-layout file containing the version of this image-layout.
+//	One index.json file listing descriptors for the contained images.
+//
 // Under blobs/, there is, for each image:
-//   One file for each layer, named after the layer's SHA.
-//   One file for each config blob, named after its SHA.
-//   One file for each manifest blob, named after its SHA.
+//
+//	One file for each layer, named after the layer's SHA.
+//	One file for each config blob, named after its SHA.
+//	One file for each manifest blob, named after its SHA.
 func Write(path string, ii v1.ImageIndex) (Path, error) {
 	lp := Path(path)
 	// Always just write oci-layout file, since it's small.