diff --git a/pkg/compiler/backend/simpledocker.go b/pkg/compiler/backend/simpledocker.go index c89161fc..966167ee 100644 --- a/pkg/compiler/backend/simpledocker.go +++ b/pkg/compiler/backend/simpledocker.go @@ -58,6 +58,36 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error { return nil } +func (*SimpleDocker) CopyImage(src, dst string) error { + Spinner(22) + defer SpinnerStop() + + Debug("Tagging image - running docker with: ", src, dst) + cmd := exec.Command("docker", "tag", src, dst) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrap(err, "Failed tagging image: "+string(out)) + } + Info(string(out)) + return nil +} + +func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error { + name := opts.ImageName + buildarg := []string{"pull", name} + Spinner(22) + defer SpinnerStop() + + Debug("Downloading image "+name+" - running docker with: ", buildarg) + cmd := exec.Command("docker", buildarg...) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrap(err, "Failed building image: "+string(out)) + } + Info(string(out)) + return nil +} + func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error { name := opts.ImageName buildarg := []string{"rmi", name} diff --git a/pkg/compiler/backend/simpleimg.go b/pkg/compiler/backend/simpleimg.go index 282141a0..70a2ef07 100644 --- a/pkg/compiler/backend/simpleimg.go +++ b/pkg/compiler/backend/simpleimg.go @@ -62,6 +62,36 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error { return nil } +func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error { + + name := opts.ImageName + buildarg := []string{"pull", name} + Spinner(22) + defer SpinnerStop() + + Debug("Downloading image "+name+" - running img with: ", buildarg) + cmd := exec.Command("img", buildarg...) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrap(err, "Failed building image: "+string(out)) + } + Info(string(out)) + return nil +} +func (*SimpleImg) CopyImage(src, dst string) error { + Spinner(22) + defer SpinnerStop() + + Debug("Tagging image - running img with: ", src, dst) + cmd := exec.Command("img", "tag", src, dst) + out, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrap(err, "Failed tagging image: "+string(out)) + } + Info(string(out)) + return nil +} + func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error { if err := s.BuildImage(opts); err != nil { return errors.Wrap(err, "Failed building image") diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 9ccaff0d..2e2415eb 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -188,6 +188,47 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage return artifact, nil } +func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions bool) (Artifact, error) { + builderOpts := CompilerBackendOptions{ + ImageName: p.GetImage(), + Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), + } + err := cs.Backend.DownloadImage(builderOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not download image") + } + + if tag != "" { + err = cs.Backend.CopyImage(p.GetImage(), tag) + if err != nil { + return nil, errors.Wrap(err, "Could not download image") + } + } + err = cs.Backend.ExportImage(builderOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not export image") + } + + rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") + if err != nil { + return nil, errors.Wrap(err, "Could not create tempdir") + } + defer os.RemoveAll(rootfs) // clean up + + // TODO: Compression and such + err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: builderOpts.Destination, Destination: rootfs}, keepPermissions) + if err != nil { + return nil, errors.Wrap(err, "Could not extract rootfs") + } + + err = helpers.Tar(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar")) + if err != nil { + return nil, errors.Wrap(err, "Error met while creating package archive") + } + return NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")), nil + +} + func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { Debug("Compiling " + p.GetPackage().GetName()) @@ -205,6 +246,10 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above if p.GetImage() != "" { + if p.ImageUnpack() { // If it is just an entire image, create a package from it + return cs.packageFromImage(p, "", keepPermissions) + } + return cs.compileWithImage(p.GetImage(), "", "", concurrency, keepPermissions, p) } @@ -261,6 +306,17 @@ func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p Compila lastHash = currentPackageImageHash if compileSpec.GetImage() != "" { + // TODO: Refactor this + if p.ImageUnpack() { // If it is just an entire image, create a package from it + artifact, err := cs.packageFromImage(p, currentPackageImageHash, keepPermissions) + if err != nil { + deperrs = append(deperrs, err) + break // stop at first error + } + departifacts = append(departifacts, artifact) + continue + } + Debug("(" + p.GetPackage().GetName() + ") Compiling " + compileSpec.GetPackage().GetFingerPrint() + " from image") artifact, err := cs.compileWithImage(compileSpec.GetImage(), buildImageHash, currentPackageImageHash, concurrency, keepPermissions, compileSpec) if err != nil { diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index d5026aad..44027442 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -144,6 +144,64 @@ var _ = Describe("Compiler", func() { Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) } + Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue()) + Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue()) + + content1, err := helpers.Read(spec.Rel("c")) + Expect(err).ToNot(HaveOccurred()) + content2, err := helpers.Read(spec.Rel("cd")) + Expect(err).ToNot(HaveOccurred()) + Expect(content1).To(Equal("c\n")) + Expect(content2).To(Equal("c\n")) + + content1, err = helpers.Read(spec.Rel("d")) + Expect(err).ToNot(HaveOccurred()) + content2, err = helpers.Read(spec.Rel("dd")) + Expect(err).ToNot(HaveOccurred()) + Expect(content1).To(Equal("s\n")) + Expect(content2).To(Equal("dd\n")) + }) + + It("unpacks images when needed", func() { + generalRecipe := tree.NewCompilerRecipe() + tmpdir, err := ioutil.TempDir("", "package") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + err = generalRecipe.Load("../../tests/fixtures/layers") + Expect(err).ToNot(HaveOccurred()) + Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point + + Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(2)) + + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.Tree()) + spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"}) + Expect(err).ToNot(HaveOccurred()) + spec2, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "base", Category: "layer", Version: "0.2"}) + Expect(err).ToNot(HaveOccurred()) + spec.SetOutputPath(tmpdir) + spec2.SetOutputPath(tmpdir) + + artifacts, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec}) + Expect(errs).To(BeNil()) + Expect(len(artifacts)).To(Equal(1)) + + artifacts2, errs := compiler.CompileParallel(1, false, []CompilationSpec{spec2}) + Expect(errs).To(BeNil()) + Expect(len(artifacts2)).To(Equal(1)) + + for _, artifact := range artifacts { + Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) + Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + } + + for _, artifact := range artifacts2 { + Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) + Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + } + + Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue()) + Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue()) }) }) }) diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go index f989d8fb..9845bf76 100644 --- a/pkg/compiler/interface.go +++ b/pkg/compiler/interface.go @@ -42,6 +42,9 @@ type CompilerBackend interface { Changes(fromImage, toImage string) ([]ArtifactLayer, error) ImageDefinitionToTar(CompilerBackendOptions) error ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error + + CopyImage(string, string) error + DownloadImage(opts CompilerBackendOptions) error } type Artifact interface { @@ -66,6 +69,8 @@ type ArtifactLayer struct { // CompilationSpec represent a compilation specification derived from a package type CompilationSpec interface { + ImageUnpack() bool // tells if the definition is just an image + RenderBuildImage() (string, error) WriteBuildImageDefinition(string) error diff --git a/pkg/compiler/spec.go b/pkg/compiler/spec.go index 201b525f..42b0caab 100644 --- a/pkg/compiler/spec.go +++ b/pkg/compiler/spec.go @@ -30,6 +30,7 @@ type LuetCompilationSpec struct { Seed string `json:"seed"` Package pkg.Package `json:"-"` OutputPath string `json:"-"` // Where the build processfiles go + Unpack bool `json:"unpack"` } func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) { @@ -50,6 +51,10 @@ func (cs *LuetCompilationSpec) BuildSteps() []string { return cs.Steps } +func (cs *LuetCompilationSpec) ImageUnpack() bool { + return cs.Unpack +} + func (cs *LuetCompilationSpec) GetPreBuildSteps() []string { return cs.Prelude } diff --git a/tests/fixtures/layers/alpine-dep/build.yaml b/tests/fixtures/layers/alpine-dep/build.yaml new file mode 100644 index 00000000..15ffba68 --- /dev/null +++ b/tests/fixtures/layers/alpine-dep/build.yaml @@ -0,0 +1,6 @@ +requires: + - category: "layer" + name: "base" + version: "0.2" +steps: + - echo alpine > /test1 \ No newline at end of file diff --git a/tests/fixtures/layers/alpine-dep/definition.yaml b/tests/fixtures/layers/alpine-dep/definition.yaml new file mode 100644 index 00000000..34a607fe --- /dev/null +++ b/tests/fixtures/layers/alpine-dep/definition.yaml @@ -0,0 +1,3 @@ +category: "layer" +name: "extra" +version: "1.0" diff --git a/tests/fixtures/layers/alpine-dep/generate.sh b/tests/fixtures/layers/alpine-dep/generate.sh new file mode 100644 index 00000000..b67417f6 --- /dev/null +++ b/tests/fixtures/layers/alpine-dep/generate.sh @@ -0,0 +1 @@ +echo generated > /artifact42 diff --git a/tests/fixtures/layers/alpine/build.yaml b/tests/fixtures/layers/alpine/build.yaml new file mode 100644 index 00000000..266699b3 --- /dev/null +++ b/tests/fixtures/layers/alpine/build.yaml @@ -0,0 +1,2 @@ +image: "alpine" +unpack: true \ No newline at end of file diff --git a/tests/fixtures/layers/alpine/definition.yaml b/tests/fixtures/layers/alpine/definition.yaml new file mode 100644 index 00000000..d969de19 --- /dev/null +++ b/tests/fixtures/layers/alpine/definition.yaml @@ -0,0 +1,3 @@ +category: "layer" +name: "base" +version: "0.2"