Compute image diffs internally

Is it more faster in this way as we already have all the needed folders
to the comparison extracted. In this way we don't repeat I/O operation
twice by calling container-diff.

Do not depend on container-diff anymore

pkg/compiler/artifact.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	system "github.com/Luet-lab/moby/pkg/system"
 	gzip "github.com/klauspost/pgzip"
 
 	//"strconv"
@@ -477,6 +478,27 @@ type CopyJob struct {
 	Artifact string
 }
 
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
@@ -490,10 +512,13 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 		// 	continue
 		// }
 
-		if !helpers.Exists(job.Dst) {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			fmt.Println("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
+			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }
@@ -550,19 +575,33 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
 					}
 				}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	} else {
 		// Otherwise just grab all
 		for _, l := range layers {
 			// Consider d.Additions (and d.Changes? - warn at least) only
 			for _, a := range l.Diffs.Additions {
+				Debug("File ", a.Name, " added")
 				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
 			}
+			for _, a := range l.Diffs.Changes {
+				Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				Debug("File ", a.Name, " deleted")
+			}
 		}
 	}
 
 	close(toCopy)
 	wg.Wait()
+
 	a := NewPackageArtifact(dst)
 	a.SetCompressionType(t)
 	err = a.Compress(archive, concurrency)

pkg/compiler/backend/diff.go

@@ -0,0 +1,199 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/config"
+	"github.com/pkg/errors"
+)
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(b compiler.CompilerBackend, srcImage, dstImage string) ([]compiler.ArtifactLayer, error) {
+
+	res := compiler.ArtifactLayer{FromImage: srcImage, ToImage: dstImage}
+
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(srcImage, "/") {
+		srcImageTar, err := ioutil.TempFile(tmpdiffs, "srctar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(srcImageTar.Name()) // clean up
+		srcImageExport := compiler.CompilerBackendOptions{
+			ImageName:   srcImage,
+			Destination: srcImageTar.Name(),
+		}
+		err = b.ExportImage(srcImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting src image "+srcImage)
+		}
+		srcImage = srcImageTar.Name()
+	}
+
+	srcImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  srcImage,
+		Destination: srcRootFS,
+	}
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+srcImage)
+	}
+
+	// Handle both files (.tar) or images. If parameters are beginning with / , don't export the images
+	if !strings.HasPrefix(dstImage, "/") {
+		dstImageTar, err := ioutil.TempFile(tmpdiffs, "dsttar")
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+
+		defer os.Remove(dstImageTar.Name()) // clean up
+		dstImageExport := compiler.CompilerBackendOptions{
+			ImageName:   dstImage,
+			Destination: dstImageTar.Name(),
+		}
+		err = b.ExportImage(dstImageExport)
+		if err != nil {
+			return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while exporting dst image "+dstImage)
+		}
+		dstImage = dstImageTar.Name()
+	}
+
+	dstImageExtract := compiler.CompilerBackendOptions{
+		SourcePath:  dstImage,
+		Destination: dstRootFS,
+	}
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+dstImage)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	return []compiler.ArtifactLayer{res}, nil
+}

pkg/compiler/backend/diff_test.go

@@ -0,0 +1,59 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend_test
+
+import (
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker backend", func() {
+	var b compiler.CompilerBackend
+
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend()
+	})
+
+	Context("Simple Docker backend satisfies main interface functionalities", func() {
+		It("Builds and generate tars", func() {
+			layers, err := GenerateChanges(b, "alpine:latest", "alpine:latest")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Builds and generate tars", func() {
+			layers, err := GenerateChanges(b, "quay.io/mocaccino/micro", "quay.io/mocaccino/extra")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/backend/simpledocker.go

@@ -16,7 +16,6 @@
 package backend
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -222,64 +221,9 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 	return nil
 }
 
-// 	container-diff diff daemon://luet/base alpine --type=file -j
+// Changes retrieves changes between image layers
-// [
+func (d *SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-//   {
+	diffs, err := GenerateChanges(d, fromImage, toImage)
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-	var errorBuffer bytes.Buffer
-
-	diffargs := []string{"diff", fromImage, toImage, "-v", "error", "-q", "--type=file", "-j", "-n", "-c", tmpdiffs}
-	cmd := exec.Command("container-diff", diffargs...)
-	cmd.Stderr = &errorBuffer
-	out, err := cmd.Output()
-
-	if string(errorBuffer.Bytes()) != "" {
-		Warning("container-diff errored with: " + string(errorBuffer.Bytes()))
-	}
-
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	if config.LuetCfg.GetGeneral().ShowBuildOutput {
-		Info(string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
 
 	if config.LuetCfg.GetGeneral().Debug {
 		summary := compiler.ComputeArtifactLayerSummary(diffs)
@@ -292,5 +236,5 @@ func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLaye
 		}
 	}
 
-	return diffs, nil
+	return diffs, err
 }

pkg/compiler/backend/simpleimg.go

@@ -149,8 +149,8 @@ func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms
 
 // TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
+func (i *SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+	return GenerateChanges(i, fromImage, toImage)
 }
 
 func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {

pkg/compiler/compiler.go

@@ -372,7 +372,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	}
 	//	}
 
-	var diffs []ArtifactLayer
 	var artifact Artifact
 	unpack := p.ImageUnpack()
 
@@ -382,14 +381,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		unpack = true
 	}
 
-	if !unpack {
-		// we have to get diffs only if spec is not unpacked
-		diffs, err = cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
-		if err != nil {
-			return nil, errors.Wrap(err, "Could not generate changes from layers")
-		}
-	}
-
 	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
@@ -432,6 +423,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		}
 		artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
 		artifact.SetCompressionType(cs.CompressionType)
+
 		err = artifact.Compress(rootfs, concurrency)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating package archive")
@@ -440,7 +432,10 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 		artifact.SetCompileSpec(p)
 	} else {
 		Info(pkgTag, "Generating delta")
-
+		diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not generate changes from layers")
+		}
 		artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), cs.CompressionType)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not generate deltas")