Add unpack field to specify a package used as layer

Closes #9

pkg/compiler/backend/simpledocker.go

@@ -58,6 +58,36 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleDocker) CopyImage(src, dst string) error {
+	Spinner(22)
+	defer SpinnerStop()
+
+	Debug("Tagging image - running docker with: ", src, dst)
+	cmd := exec.Command("docker", "tag", src, dst)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed tagging image: "+string(out))
+	}
+	Info(string(out))
+	return nil
+}
+
+func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
+	name := opts.ImageName
+	buildarg := []string{"pull", name}
+	Spinner(22)
+	defer SpinnerStop()
+
+	Debug("Downloading image "+name+" - running docker with: ", buildarg)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed building image: "+string(out))
+	}
+	Info(string(out))
+	return nil
+}
+
 func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}

pkg/compiler/backend/simpleimg.go

@@ -62,6 +62,36 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	return nil
 }
 
+func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
+
+	name := opts.ImageName
+	buildarg := []string{"pull", name}
+	Spinner(22)
+	defer SpinnerStop()
+
+	Debug("Downloading image "+name+" - running img with: ", buildarg)
+	cmd := exec.Command("img", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed building image: "+string(out))
+	}
+	Info(string(out))
+	return nil
+}
+func (*SimpleImg) CopyImage(src, dst string) error {
+	Spinner(22)
+	defer SpinnerStop()
+
+	Debug("Tagging image - running img with: ", src, dst)
+	cmd := exec.Command("img", "tag", src, dst)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed tagging image: "+string(out))
+	}
+	Info(string(out))
+	return nil
+}
+
 func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")

pkg/compiler/compiler.go

@@ -188,6 +188,47 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 	return artifact, nil
 }
 
+func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) {
+	builderOpts := CompilerBackendOptions{
+		ImageName:   p.GetImage(),
+		Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
+	}
+	err := cs.Backend.DownloadImage(builderOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not download image")
+	}
+
+	if tag != "" {
+		err = cs.Backend.CopyImage(p.GetImage(), tag)
+		if err != nil {
+			return nil, errors.Wrap(err, "Could not download image")
+		}
+	}
+	err = cs.Backend.ExportImage(builderOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not export image")
+	}
+
+	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not create tempdir")
+	}
+	defer os.RemoveAll(rootfs) // clean up
+
+	// TODO: Compression and such
+	err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions)
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not extract rootfs")
+	}
+
+	err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"))
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+	return NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")), nil
+
+}
+
 func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
 	Debug("Compiling " + p.GetPackage().GetName())
 
@@ -205,6 +246,10 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
+		if p.ImageUnpack() { // If it is just an entire image, create a package from it
+			return cs.packageFromImage(p, "", keepPermissions)
+		}
+
 		return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p)
 	}
 
@@ -261,6 +306,17 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila
 
 			lastHash = currentPackageImageHash
 			if compileSpec.GetImage() != "" {
+				// TODO: Refactor this
+				if p.ImageUnpack() { // If it is just an entire image, create a package from it
+					artifact, err := cs.packageFromImage(p, currentPackageImageHash, keepPermissions)
+					if err != nil {
+						deperrs = append(deperrs, err)
+						break // stop at first error
+					}
+					departifacts = append(departifacts, artifact)
+					continue
+				}
+
 				Debug("(" + p.GetPackage().GetName() + ") Compiling " + compileSpec.GetPackage().GetFingerPrint() + " from image")
 				artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec)
 				if err != nil {

pkg/compiler/compiler_test.go

@@ -144,6 +144,64 @@ var _ = Describe("Compiler", func() {
 				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
 			}
 
+			Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
+
+			content1, err := helpers.Read(spec.Rel("c"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err := helpers.Read(spec.Rel("cd"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("c\n"))
+			Expect(content2).To(Equal("c\n"))
+
+			content1, err = helpers.Read(spec.Rel("d"))
+			Expect(err).ToNot(HaveOccurred())
+			content2, err = helpers.Read(spec.Rel("dd"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("s\n"))
+			Expect(content2).To(Equal("dd\n"))
+		})
+
+		It("unpacks images when needed", func() {
+			generalRecipe := tree.NewCompilerRecipe()
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/layers")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
+
+			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(2))
+
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree())
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+			spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"})
+			Expect(err).ToNot(HaveOccurred())
+			spec.SetOutputPath(tmpdir)
+			spec2.SetOutputPath(tmpdir)
+
+			artifacts, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec})
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts)).To(Equal(1))
+
+			artifacts2, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec2})
+			Expect(errs).To(BeNil())
+			Expect(len(artifacts2)).To(Equal(1))
+
+			for _, artifact := range artifacts {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+
+			for _, artifact := range artifacts2 {
+				Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
+				Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
+			}
+
+			Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
+			Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
 		})
 	})
 })

pkg/compiler/interface.go

@@ -42,6 +42,9 @@ type CompilerBackend interface {
 	Changes(fromImage, toImage string) ([]ArtifactLayer, error)
 	ImageDefinitionToTar(CompilerBackendOptions) error
 	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
+
+	CopyImage(string, string) error
+	DownloadImage(opts CompilerBackendOptions) error
 }
 
 type Artifact interface {
@@ -66,6 +69,8 @@ type ArtifactLayer struct {
 
 // CompilationSpec represent a compilation specification derived from a package
 type CompilationSpec interface {
+	ImageUnpack() bool // tells if the definition is just an image
+
 	RenderBuildImage() (string, error)
 	WriteBuildImageDefinition(string) error
 

pkg/compiler/spec.go

@@ -30,6 +30,7 @@ type LuetCompilationSpec struct {
 	Seed       string      `json:"seed"`
 	Package    pkg.Package `json:"-"`
 	OutputPath string      `json:"-"` // Where the build processfiles go
+	Unpack     bool        `json:"unpack"`
 }
 
 func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
@@ -50,6 +51,10 @@ func (cs *LuetCompilationSpec) BuildSteps() []string {
 	return cs.Steps
 }
 
+func (cs *LuetCompilationSpec) ImageUnpack() bool {
+	return cs.Unpack
+}
+
 func (cs *LuetCompilationSpec) GetPreBuildSteps() []string {
 	return cs.Prelude
 }

tests/fixtures/layers/alpine-dep/build.yaml

@@ -0,0 +1,6 @@
+requires:
+  - category: "layer"
+    name: "base"
+    version: "0.2"
+steps:
+  - echo alpine > /test1

tests/fixtures/layers/alpine-dep/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "extra"
+version: "1.0"

tests/fixtures/layers/alpine-dep/generate.sh

@@ -0,0 +1 @@
+echo generated > /artifact42

tests/fixtures/layers/alpine/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/layers/alpine/definition.yaml

@@ -0,0 +1,3 @@
+category: "layer"
+name: "base"
+version: "0.2"