Update gomod and vendor

vendor/github.com/moby/buildkit/cache/remotecache/v1/cachestorage.go

@@ -0,0 +1,297 @@
+package cacheimport
+
+import (
+	"context"
+	"time"
+
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/worker"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+func NewCacheKeyStorage(cc *CacheChains, w worker.Worker) (solver.CacheKeyStorage, solver.CacheResultStorage, error) {
+	storage := &cacheKeyStorage{
+		byID:     map[string]*itemWithOutgoingLinks{},
+		byItem:   map[*item]string{},
+		byResult: map[string]map[string]struct{}{},
+	}
+
+	for _, it := range cc.items {
+		if _, err := addItemToStorage(storage, it); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	results := &cacheResultStorage{
+		w:        w,
+		byID:     storage.byID,
+		byItem:   storage.byItem,
+		byResult: storage.byResult,
+	}
+
+	return storage, results, nil
+}
+
+func addItemToStorage(k *cacheKeyStorage, it *item) (*itemWithOutgoingLinks, error) {
+	if id, ok := k.byItem[it]; ok {
+		if id == "" {
+			return nil, errors.Errorf("invalid loop")
+		}
+		return k.byID[id], nil
+	}
+
+	var id string
+	if len(it.links) == 0 {
+		id = it.dgst.String()
+	} else {
+		id = identity.NewID()
+	}
+
+	k.byItem[it] = ""
+
+	for i, m := range it.links {
+		for l := range m {
+			src, err := addItemToStorage(k, l.src)
+			if err != nil {
+				return nil, err
+			}
+			cl := nlink{
+				input:    i,
+				dgst:     it.dgst,
+				selector: l.selector,
+			}
+			src.links[cl] = append(src.links[cl], id)
+		}
+	}
+
+	k.byItem[it] = id
+
+	itl := &itemWithOutgoingLinks{
+		item:  it,
+		links: map[nlink][]string{},
+	}
+
+	k.byID[id] = itl
+
+	if res := it.result; res != nil {
+		resultID := remoteID(res)
+		ids, ok := k.byResult[resultID]
+		if !ok {
+			ids = map[string]struct{}{}
+			k.byResult[resultID] = ids
+		}
+		ids[id] = struct{}{}
+	}
+	return itl, nil
+}
+
+type cacheKeyStorage struct {
+	byID     map[string]*itemWithOutgoingLinks
+	byItem   map[*item]string
+	byResult map[string]map[string]struct{}
+}
+
+type itemWithOutgoingLinks struct {
+	*item
+	links map[nlink][]string
+}
+
+func (cs *cacheKeyStorage) Exists(id string) bool {
+	_, ok := cs.byID[id]
+	return ok
+}
+
+func (cs *cacheKeyStorage) Walk(func(id string) error) error {
+	return nil
+}
+
+func (cs *cacheKeyStorage) WalkResults(id string, fn func(solver.CacheResult) error) error {
+	it, ok := cs.byID[id]
+	if !ok {
+		return nil
+	}
+	if res := it.result; res != nil {
+		return fn(solver.CacheResult{ID: remoteID(res), CreatedAt: it.resultTime})
+	}
+	return nil
+}
+
+func (cs *cacheKeyStorage) Load(id string, resultID string) (solver.CacheResult, error) {
+	it, ok := cs.byID[id]
+	if !ok {
+		return solver.CacheResult{}, nil
+	}
+	if res := it.result; res != nil {
+		return solver.CacheResult{ID: remoteID(res), CreatedAt: it.resultTime}, nil
+	}
+	return solver.CacheResult{}, nil
+}
+
+func (cs *cacheKeyStorage) AddResult(id string, res solver.CacheResult) error {
+	return nil
+}
+
+func (cs *cacheKeyStorage) Release(resultID string) error {
+	return nil
+}
+func (cs *cacheKeyStorage) AddLink(id string, link solver.CacheInfoLink, target string) error {
+	return nil
+}
+func (cs *cacheKeyStorage) WalkLinks(id string, link solver.CacheInfoLink, fn func(id string) error) error {
+	it, ok := cs.byID[id]
+	if !ok {
+		return nil
+	}
+	for _, id := range it.links[nlink{
+		dgst:     outputKey(link.Digest, int(link.Output)),
+		input:    int(link.Input),
+		selector: link.Selector.String(),
+	}] {
+		if err := fn(id); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cs *cacheKeyStorage) WalkBacklinks(id string, fn func(id string, link solver.CacheInfoLink) error) error {
+	for k, it := range cs.byID {
+		for nl, ids := range it.links {
+			for _, id2 := range ids {
+				if id == id2 {
+					if err := fn(k, solver.CacheInfoLink{
+						Input:    solver.Index(nl.input),
+						Selector: digest.Digest(nl.selector),
+						Digest:   nl.dgst,
+					}); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (cs *cacheKeyStorage) WalkIDsByResult(id string, fn func(id string) error) error {
+	ids := cs.byResult[id]
+	for id := range ids {
+		if err := fn(id); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cs *cacheKeyStorage) HasLink(id string, link solver.CacheInfoLink, target string) bool {
+	l := nlink{
+		dgst:     outputKey(link.Digest, int(link.Output)),
+		input:    int(link.Input),
+		selector: link.Selector.String(),
+	}
+	if it, ok := cs.byID[id]; ok {
+		for _, id := range it.links[l] {
+			if id == target {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+type cacheResultStorage struct {
+	w        worker.Worker
+	byID     map[string]*itemWithOutgoingLinks
+	byResult map[string]map[string]struct{}
+	byItem   map[*item]string
+}
+
+func (cs *cacheResultStorage) Save(res solver.Result, createdAt time.Time) (solver.CacheResult, error) {
+	return solver.CacheResult{}, errors.Errorf("importer is immutable")
+}
+
+func (cs *cacheResultStorage) LoadWithParents(ctx context.Context, res solver.CacheResult) (map[string]solver.Result, error) {
+	v := cs.byResultID(res.ID)
+	if v == nil || v.result == nil {
+		return nil, errors.WithStack(solver.ErrNotFound)
+	}
+
+	m := map[string]solver.Result{}
+
+	visited := make(map[*item]struct{})
+	if err := v.walkAllResults(func(i *item) error {
+		if i.result == nil {
+			return nil
+		}
+		id, ok := cs.byItem[i]
+		if !ok {
+			return nil
+		}
+		if isSubRemote(*i.result, *v.result) {
+			ref, err := cs.w.FromRemote(ctx, i.result)
+			if err != nil {
+				return err
+			}
+			m[id] = worker.NewWorkerRefResult(ref, cs.w)
+		}
+		return nil
+	}, visited); err != nil {
+		for _, v := range m {
+			v.Release(context.TODO())
+		}
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func (cs *cacheResultStorage) Load(ctx context.Context, res solver.CacheResult) (solver.Result, error) {
+	item := cs.byResultID(res.ID)
+	if item == nil || item.result == nil {
+		return nil, errors.WithStack(solver.ErrNotFound)
+	}
+
+	ref, err := cs.w.FromRemote(ctx, item.result)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load result from remote")
+	}
+	return worker.NewWorkerRefResult(ref, cs.w), nil
+}
+
+func (cs *cacheResultStorage) LoadRemote(ctx context.Context, res solver.CacheResult) (*solver.Remote, error) {
+	if r := cs.byResultID(res.ID); r != nil && r.result != nil {
+		return r.result, nil
+	}
+	return nil, errors.WithStack(solver.ErrNotFound)
+}
+
+func (cs *cacheResultStorage) Exists(id string) bool {
+	return cs.byResultID(id) != nil
+}
+
+func (cs *cacheResultStorage) byResultID(resultID string) *itemWithOutgoingLinks {
+	m, ok := cs.byResult[resultID]
+	if !ok || len(m) == 0 {
+		return nil
+	}
+
+	for id := range m {
+		it, ok := cs.byID[id]
+		if ok {
+			return it
+		}
+	}
+
+	return nil
+}
+
+// unique ID per remote. this ID is not stable.
+func remoteID(r *solver.Remote) string {
+	dgstr := digest.Canonical.Digester()
+	for _, desc := range r.Descriptors {
+		dgstr.Hash().Write([]byte(desc.Digest))
+	}
+	return dgstr.Digest().String()
+}

vendor/github.com/moby/buildkit/cache/remotecache/v1/chains.go

@@ -0,0 +1,158 @@
+package cacheimport
+
+import (
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/moby/buildkit/solver"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func NewCacheChains() *CacheChains {
+	return &CacheChains{visited: map[interface{}]struct{}{}}
+}
+
+type CacheChains struct {
+	items   []*item
+	visited map[interface{}]struct{}
+}
+
+func (c *CacheChains) Add(dgst digest.Digest) solver.CacheExporterRecord {
+	if strings.HasPrefix(dgst.String(), "random:") {
+		return &nopRecord{}
+	}
+	it := &item{c: c, dgst: dgst}
+	c.items = append(c.items, it)
+	return it
+}
+
+func (c *CacheChains) Visit(v interface{}) {
+	c.visited[v] = struct{}{}
+}
+
+func (c *CacheChains) Visited(v interface{}) bool {
+	_, ok := c.visited[v]
+	return ok
+}
+
+func (c *CacheChains) normalize() error {
+	st := &normalizeState{
+		added: map[*item]*item{},
+		links: map[*item]map[nlink]map[digest.Digest]struct{}{},
+		byKey: map[digest.Digest]*item{},
+	}
+
+	for _, it := range c.items {
+		_, err := normalizeItem(it, st)
+		if err != nil {
+			return err
+		}
+	}
+
+	items := make([]*item, 0, len(st.byKey))
+	for _, it := range st.byKey {
+		items = append(items, it)
+	}
+	c.items = items
+	return nil
+}
+
+func (c *CacheChains) Marshal() (*CacheConfig, DescriptorProvider, error) {
+	if err := c.normalize(); err != nil {
+		return nil, nil, err
+	}
+
+	st := &marshalState{
+		chainsByID:    map[string]int{},
+		descriptors:   DescriptorProvider{},
+		recordsByItem: map[*item]int{},
+	}
+
+	for _, it := range c.items {
+		if err := marshalItem(it, st); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	cc := CacheConfig{
+		Layers:  st.layers,
+		Records: st.records,
+	}
+	sortConfig(&cc)
+
+	return &cc, st.descriptors, nil
+}
+
+type DescriptorProvider map[digest.Digest]DescriptorProviderPair
+
+type DescriptorProviderPair struct {
+	Descriptor ocispec.Descriptor
+	Provider   content.Provider
+}
+
+type item struct {
+	c    *CacheChains
+	dgst digest.Digest
+
+	result     *solver.Remote
+	resultTime time.Time
+
+	links []map[link]struct{}
+}
+
+type link struct {
+	src      *item
+	selector string
+}
+
+func (c *item) AddResult(createdAt time.Time, result *solver.Remote) {
+	c.resultTime = createdAt
+	c.result = result
+}
+
+func (c *item) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) {
+	src, ok := rec.(*item)
+	if !ok {
+		return
+	}
+
+	for {
+		if index < len(c.links) {
+			break
+		}
+		c.links = append(c.links, map[link]struct{}{})
+	}
+
+	c.links[index][link{src: src, selector: selector}] = struct{}{}
+}
+
+func (c *item) walkAllResults(fn func(i *item) error, visited map[*item]struct{}) error {
+	if _, ok := visited[c]; ok {
+		return nil
+	}
+	visited[c] = struct{}{}
+	if err := fn(c); err != nil {
+		return err
+	}
+	for _, links := range c.links {
+		for l := range links {
+			if err := l.src.walkAllResults(fn, visited); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+type nopRecord struct {
+}
+
+func (c *nopRecord) AddResult(createdAt time.Time, result *solver.Remote) {
+}
+
+func (c *nopRecord) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) {
+}
+
+var _ solver.CacheExporterTarget = &CacheChains{}

vendor/github.com/moby/buildkit/cache/remotecache/v1/doc.go

@@ -0,0 +1,50 @@
+package cacheimport
+
+// Distibutable build cache
+//
+// Main manifest is OCI image index
+// https://github.com/opencontainers/image-spec/blob/master/image-index.md .
+// Manifests array contains descriptors to the cache layers and one instance of
+// build cache config with media type application/vnd.buildkit.cacheconfig.v0 .
+// The cache layer descriptors need to have an annotation with uncompressed digest
+// to allow deduplication on extraction and optionally "buildkit/createdat"
+// annotation to support maintaining original timestamps.
+//
+// Cache config file layout:
+//
+//{
+//  "layers": [
+//    {
+//      "blob": "sha256:deadbeef",    <- digest of layer blob in index
+//      "parent": -1                  <- index of parent layer, -1 if no parent
+//    },
+//    {
+//      "blob": "sha256:deadbeef",
+//      "parent": 0
+//    }
+//  ],
+//
+//  "records": [
+//    {
+//      "digest": "sha256:deadbeef",   <- base digest for the record
+//    },
+//    {
+//      "digest": "sha256:deadbeef",
+//      "output": 1,                   <- optional output index
+//      "layers": [                    <- optional array or layer chains
+//        {
+//          "createdat": "",
+//          "layer": 1,                <- index to the layer
+//        }
+//      ],
+//      "inputs": [                    <- dependant records
+//        [                            <- index of the dependency (0)
+//          {
+//            "selector": "sel",       <- optional selector
+//            "link": 0,               <- index to the dependant record
+//          }
+//        ]
+//      ]
+//    }
+//  ]
+// }

vendor/github.com/moby/buildkit/cache/remotecache/v1/parse.go

@@ -0,0 +1,110 @@
+package cacheimport
+
+import (
+	"encoding/json"
+
+	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/util/contentutil"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func Parse(configJSON []byte, provider DescriptorProvider, t solver.CacheExporterTarget) error {
+	var config CacheConfig
+	if err := json.Unmarshal(configJSON, &config); err != nil {
+		return errors.WithStack(err)
+	}
+
+	return ParseConfig(config, provider, t)
+}
+
+func ParseConfig(config CacheConfig, provider DescriptorProvider, t solver.CacheExporterTarget) error {
+	cache := map[int]solver.CacheExporterRecord{}
+
+	for i := range config.Records {
+		if _, err := parseRecord(config, i, provider, t, cache); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver.CacheExporterTarget, cache map[int]solver.CacheExporterRecord) (solver.CacheExporterRecord, error) {
+	if r, ok := cache[idx]; ok {
+		if r == nil {
+			return nil, errors.Errorf("invalid looping record")
+		}
+		return r, nil
+	}
+
+	if idx < 0 || idx >= len(cc.Records) {
+		return nil, errors.Errorf("invalid record ID: %d", idx)
+	}
+	rec := cc.Records[idx]
+
+	r := t.Add(rec.Digest)
+	cache[idx] = nil
+	for i, inputs := range rec.Inputs {
+		for _, inp := range inputs {
+			src, err := parseRecord(cc, inp.LinkIndex, provider, t, cache)
+			if err != nil {
+				return nil, err
+			}
+			r.LinkFrom(src, i, inp.Selector)
+		}
+	}
+
+	for _, res := range rec.Results {
+		visited := map[int]struct{}{}
+		remote, err := getRemoteChain(cc.Layers, res.LayerIndex, provider, visited)
+		if err != nil {
+			return nil, err
+		}
+		if remote != nil {
+			r.AddResult(res.CreatedAt, remote)
+		}
+	}
+
+	cache[idx] = r
+	return r, nil
+}
+
+func getRemoteChain(layers []CacheLayer, idx int, provider DescriptorProvider, visited map[int]struct{}) (*solver.Remote, error) {
+	if _, ok := visited[idx]; ok {
+		return nil, errors.Errorf("invalid looping layer")
+	}
+	visited[idx] = struct{}{}
+
+	if idx < 0 || idx >= len(layers) {
+		return nil, errors.Errorf("invalid layer index %d", idx)
+	}
+
+	l := layers[idx]
+
+	descPair, ok := provider[l.Blob]
+	if !ok {
+		return nil, nil
+	}
+
+	var r *solver.Remote
+	if l.ParentIndex != -1 {
+		var err error
+		r, err = getRemoteChain(layers, l.ParentIndex, provider, visited)
+		if err != nil {
+			return nil, err
+		}
+		if r == nil {
+			return nil, nil
+		}
+		r.Descriptors = append(r.Descriptors, descPair.Descriptor)
+		mp := contentutil.NewMultiProvider(r.Provider)
+		mp.Add(descPair.Descriptor.Digest, descPair.Provider)
+		r.Provider = mp
+		return r, nil
+	}
+	return &solver.Remote{
+		Descriptors: []ocispec.Descriptor{descPair.Descriptor},
+		Provider:    descPair.Provider,
+	}, nil
+
+}

vendor/github.com/moby/buildkit/cache/remotecache/v1/spec.go

@@ -0,0 +1,35 @@
+package cacheimport
+
+import (
+	"time"
+
+	digest "github.com/opencontainers/go-digest"
+)
+
+const CacheConfigMediaTypeV0 = "application/vnd.buildkit.cacheconfig.v0"
+
+type CacheConfig struct {
+	Layers  []CacheLayer  `json:"layers,omitempty"`
+	Records []CacheRecord `json:"records,omitempty"`
+}
+
+type CacheLayer struct {
+	Blob        digest.Digest `json:"blob,omitempty"`
+	ParentIndex int           `json:"parent,omitempty"`
+}
+
+type CacheRecord struct {
+	Results []CacheResult  `json:"layers,omitempty"`
+	Digest  digest.Digest  `json:"digest,omitempty"`
+	Inputs  [][]CacheInput `json:"inputs,omitempty"`
+}
+
+type CacheResult struct {
+	LayerIndex int       `json:"layer"`
+	CreatedAt  time.Time `json:"createdAt,omitempty"`
+}
+
+type CacheInput struct {
+	Selector  string `json:"selector,omitempty"`
+	LinkIndex int    `json:"link"`
+}

vendor/github.com/moby/buildkit/cache/remotecache/v1/utils.go

@@ -0,0 +1,322 @@
+package cacheimport
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/containerd/containerd/content"
+	"github.com/moby/buildkit/solver"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// sortConfig sorts the config structure to make sure it is deterministic
+func sortConfig(cc *CacheConfig) {
+	type indexedLayer struct {
+		oldIndex int
+		newIndex int
+		l        CacheLayer
+	}
+
+	unsortedLayers := make([]*indexedLayer, len(cc.Layers))
+	sortedLayers := make([]*indexedLayer, len(cc.Layers))
+
+	for i, l := range cc.Layers {
+		il := &indexedLayer{oldIndex: i, l: l}
+		unsortedLayers[i] = il
+		sortedLayers[i] = il
+	}
+	sort.Slice(sortedLayers, func(i, j int) bool {
+		li := sortedLayers[i].l
+		lj := sortedLayers[j].l
+		if li.Blob == lj.Blob {
+			return li.ParentIndex < lj.ParentIndex
+		}
+		return li.Blob < lj.Blob
+	})
+	for i, l := range sortedLayers {
+		l.newIndex = i
+	}
+
+	layers := make([]CacheLayer, len(sortedLayers))
+	for i, l := range sortedLayers {
+		if pID := l.l.ParentIndex; pID != -1 {
+			l.l.ParentIndex = unsortedLayers[pID].newIndex
+		}
+		layers[i] = l.l
+	}
+
+	type indexedRecord struct {
+		oldIndex int
+		newIndex int
+		r        CacheRecord
+	}
+
+	unsortedRecords := make([]*indexedRecord, len(cc.Records))
+	sortedRecords := make([]*indexedRecord, len(cc.Records))
+
+	for i, r := range cc.Records {
+		ir := &indexedRecord{oldIndex: i, r: r}
+		unsortedRecords[i] = ir
+		sortedRecords[i] = ir
+	}
+	sort.Slice(sortedRecords, func(i, j int) bool {
+		ri := sortedRecords[i].r
+		rj := sortedRecords[j].r
+		if ri.Digest != rj.Digest {
+			return ri.Digest < rj.Digest
+		}
+		if len(ri.Inputs) != len(rj.Inputs) {
+			return len(ri.Inputs) < len(rj.Inputs)
+		}
+		for i, inputs := range ri.Inputs {
+			if len(ri.Inputs[i]) != len(rj.Inputs[i]) {
+				return len(ri.Inputs[i]) < len(rj.Inputs[i])
+			}
+			for j := range inputs {
+				if ri.Inputs[i][j].Selector != rj.Inputs[i][j].Selector {
+					return ri.Inputs[i][j].Selector < rj.Inputs[i][j].Selector
+				}
+				inputDigesti := cc.Records[ri.Inputs[i][j].LinkIndex].Digest
+				inputDigestj := cc.Records[rj.Inputs[i][j].LinkIndex].Digest
+				if inputDigesti != inputDigestj {
+					return inputDigesti < inputDigestj
+				}
+			}
+		}
+		return false
+	})
+	for i, l := range sortedRecords {
+		l.newIndex = i
+	}
+
+	records := make([]CacheRecord, len(sortedRecords))
+	for i, r := range sortedRecords {
+		for j := range r.r.Results {
+			r.r.Results[j].LayerIndex = unsortedLayers[r.r.Results[j].LayerIndex].newIndex
+		}
+		for j, inputs := range r.r.Inputs {
+			for k := range inputs {
+				r.r.Inputs[j][k].LinkIndex = unsortedRecords[r.r.Inputs[j][k].LinkIndex].newIndex
+			}
+			sort.Slice(inputs, func(i, j int) bool {
+				return inputs[i].LinkIndex < inputs[j].LinkIndex
+			})
+		}
+		records[i] = r.r
+	}
+
+	cc.Layers = layers
+	cc.Records = records
+}
+
+func outputKey(dgst digest.Digest, idx int) digest.Digest {
+	return digest.FromBytes([]byte(fmt.Sprintf("%s@%d", dgst, idx)))
+}
+
+type nlink struct {
+	dgst     digest.Digest
+	input    int
+	selector string
+}
+type normalizeState struct {
+	added map[*item]*item
+	links map[*item]map[nlink]map[digest.Digest]struct{}
+	byKey map[digest.Digest]*item
+	next  int
+}
+
+func normalizeItem(it *item, state *normalizeState) (*item, error) {
+	if it2, ok := state.added[it]; ok {
+		return it2, nil
+	}
+
+	if len(it.links) == 0 {
+		id := it.dgst
+		if it2, ok := state.byKey[id]; ok {
+			state.added[it] = it2
+			return it2, nil
+		}
+		state.byKey[id] = it
+		state.added[it] = it
+		return nil, nil
+	}
+
+	matches := map[digest.Digest]struct{}{}
+
+	// check if there is already a matching record
+	for i, m := range it.links {
+		if len(m) == 0 {
+			return nil, errors.Errorf("invalid incomplete links")
+		}
+		for l := range m {
+			nl := nlink{dgst: it.dgst, input: i, selector: l.selector}
+			it2, err := normalizeItem(l.src, state)
+			if err != nil {
+				return nil, err
+			}
+			links := state.links[it2][nl]
+			if i == 0 {
+				for id := range links {
+					matches[id] = struct{}{}
+				}
+			} else {
+				for id := range matches {
+					if _, ok := links[id]; !ok {
+						delete(matches, id)
+					}
+				}
+			}
+		}
+	}
+
+	var id digest.Digest
+
+	links := it.links
+
+	if len(matches) > 0 {
+		for m := range matches {
+			if id == "" || id > m {
+				id = m
+			}
+		}
+	} else {
+		// keep tmp IDs deterministic
+		state.next++
+		id = digest.FromBytes([]byte(fmt.Sprintf("%d", state.next)))
+		state.byKey[id] = it
+		it.links = make([]map[link]struct{}, len(it.links))
+		for i := range it.links {
+			it.links[i] = map[link]struct{}{}
+		}
+	}
+
+	it2 := state.byKey[id]
+	state.added[it] = it2
+
+	for i, m := range links {
+		for l := range m {
+			subIt, err := normalizeItem(l.src, state)
+			if err != nil {
+				return nil, err
+			}
+			it2.links[i][link{src: subIt, selector: l.selector}] = struct{}{}
+
+			nl := nlink{dgst: it.dgst, input: i, selector: l.selector}
+			if _, ok := state.links[subIt]; !ok {
+				state.links[subIt] = map[nlink]map[digest.Digest]struct{}{}
+			}
+			if _, ok := state.links[subIt][nl]; !ok {
+				state.links[subIt][nl] = map[digest.Digest]struct{}{}
+			}
+			state.links[subIt][nl][id] = struct{}{}
+		}
+	}
+
+	return it2, nil
+}
+
+type marshalState struct {
+	layers      []CacheLayer
+	chainsByID  map[string]int
+	descriptors DescriptorProvider
+
+	records       []CacheRecord
+	recordsByItem map[*item]int
+}
+
+func marshalRemote(r *solver.Remote, state *marshalState) string {
+	if len(r.Descriptors) == 0 {
+		return ""
+	}
+	type Remote struct {
+		Descriptors []ocispec.Descriptor
+		Provider    content.Provider
+	}
+	var parentID string
+	if len(r.Descriptors) > 1 {
+		r2 := &solver.Remote{
+			Descriptors: r.Descriptors[:len(r.Descriptors)-1],
+			Provider:    r.Provider,
+		}
+		parentID = marshalRemote(r2, state)
+	}
+	desc := r.Descriptors[len(r.Descriptors)-1]
+
+	state.descriptors[desc.Digest] = DescriptorProviderPair{
+		Descriptor: desc,
+		Provider:   r.Provider,
+	}
+
+	id := desc.Digest.String() + parentID
+
+	if _, ok := state.chainsByID[id]; ok {
+		return id
+	}
+
+	state.chainsByID[id] = len(state.layers)
+	l := CacheLayer{
+		Blob:        desc.Digest,
+		ParentIndex: -1,
+	}
+	if parentID != "" {
+		l.ParentIndex = state.chainsByID[parentID]
+	}
+	state.layers = append(state.layers, l)
+	return id
+}
+
+func marshalItem(it *item, state *marshalState) error {
+	if _, ok := state.recordsByItem[it]; ok {
+		return nil
+	}
+
+	rec := CacheRecord{
+		Digest: it.dgst,
+		Inputs: make([][]CacheInput, len(it.links)),
+	}
+
+	for i, m := range it.links {
+		for l := range m {
+			if err := marshalItem(l.src, state); err != nil {
+				return err
+			}
+			idx, ok := state.recordsByItem[l.src]
+			if !ok {
+				return errors.Errorf("invalid source record: %v", l.src)
+			}
+			rec.Inputs[i] = append(rec.Inputs[i], CacheInput{
+				Selector:  l.selector,
+				LinkIndex: idx,
+			})
+		}
+	}
+
+	if it.result != nil {
+		id := marshalRemote(it.result, state)
+		if id != "" {
+			idx, ok := state.chainsByID[id]
+			if !ok {
+				return errors.Errorf("parent chainid not found")
+			}
+			rec.Results = append(rec.Results, CacheResult{LayerIndex: idx, CreatedAt: it.resultTime})
+		}
+	}
+
+	state.recordsByItem[it] = len(state.records)
+	state.records = append(state.records, rec)
+	return nil
+}
+
+func isSubRemote(sub, main solver.Remote) bool {
+	if len(sub.Descriptors) > len(main.Descriptors) {
+		return false
+	}
+	for i := range sub.Descriptors {
+		if sub.Descriptors[i].Digest != main.Descriptors[i].Digest {
+			return false
+		}
+	}
+	return true
+}