From ff88ff67c24da010a5d9e726105dbb4c557892c5 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 5 Nov 2019 17:36:22 +0100 Subject: [PATCH] Add Separate tree for build dependency Reuse the Recipe and extend it to read a separate tree for build dependencies. Also add accessors to compilespec to produce dockerfile image format. --- pkg/compiler/compiler.go | 28 ++++- pkg/compiler/interface.go | 18 ++- pkg/compiler/spec.go | 73 +++++++++++- pkg/compiler/spec_test.go | 28 +++++ pkg/helpers/file.go | 13 ++- pkg/tree/compiler_recipe.go | 105 ++++++++++++++++++ .../app-admin/enman/1.4.0/build.yaml | 1 + 7 files changed, 254 insertions(+), 12 deletions(-) create mode 100644 pkg/tree/compiler_recipe.go diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 88c621ad..943318e9 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -27,15 +27,37 @@ import ( const BuildFile = "build.yaml" type LuetCompiler struct { - *tree.Recipe + *tree.CompilerRecipe Backend CompilerBackend } func NewLuetCompiler(backend CompilerBackend, t pkg.Tree) Compiler { - return &LuetCompiler{Backend: backend, Recipe: &tree.Recipe{PackageTree: t}} + // The CompilerRecipe will gives us a tree with only build deps listed. + return &LuetCompiler{ + Backend: backend, + CompilerRecipe: &tree.CompilerRecipe{ + tree.Recipe{PackageTree: t}, + }, + } } func (cs *LuetCompiler) Compile(p CompilationSpec) (*Artifact, error) { + + // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based + // on how we compute the resolvable tree. + // This means to recursively build all the build-images needed to reach that tree part. + // - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image. + // - If image is set we just generate a plain dockerfile + + // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above + + if p.GetImage() != "" { + p.SetSeedImage(p.GetImage()) + //p.WriteBuildImageDefinition(path) + //backend.BuildImage(path) + //backend.RunSteps(CompilationSpec) + } + return nil, errors.New("Not implemented yet") } @@ -54,7 +76,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { if err != nil { return nil, err } - return NewLuetCompilationSpec(dat) + return NewLuetCompilationSpec(dat, p) } func (cs *LuetCompiler) GetBackend() CompilerBackend { diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go index ceda1c07..9dd2f8be 100644 --- a/pkg/compiler/interface.go +++ b/pkg/compiler/interface.go @@ -26,11 +26,23 @@ type Compiler interface { } type CompilerBackend interface { - Tree() pkg.Tree - WithTree(pkg.Tree) + BuildImage(name, path,dockerfileName string) error } // CompilationSpec represent a compilation specification derived from a package type CompilationSpec interface { - ToDocker() (string, error) + RenderBuildImage() (string, error) + WriteBuildImageDefinition(string) error + + RenderStepImage(image string) (string, error) + WriteStepImageDefinition(fromimage, path string) error + + GetPackage() pkg.Package + BuildSteps() []string + + GetSeedImage() string + SetSeedImage(string) + + GetImage() string + SetImage(string) } diff --git a/pkg/compiler/spec.go b/pkg/compiler/spec.go index 660b05ab..932393e7 100644 --- a/pkg/compiler/spec.go +++ b/pkg/compiler/spec.go @@ -16,23 +16,86 @@ package compiler import ( + pkg "github.com/mudler/luet/pkg/package" yaml "gopkg.in/yaml.v2" + "io/ioutil" ) type LuetCompilationSpec struct { - Steps []string `json:"steps"` - Image string `json:"image"` + Steps []string `json:"steps"` // Are run inside a container and the result layer diff is saved + Image string `json:"image"` + Seed string `json:"seed"` + Package pkg.Package `json:"-"` } -func NewLuetCompilationSpec(b []byte) (CompilationSpec, error) { +func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) { var spec LuetCompilationSpec err := yaml.Unmarshal(b, &spec) if err != nil { return &spec, err } + spec.Package = p return &spec, nil } -func (cs *LuetCompilationSpec) ToDocker() (string, error) { - return "", nil +func (cs *LuetCompilationSpec) GetPackage() pkg.Package { + return cs.Package +} + +func (cs *LuetCompilationSpec) BuildSteps() []string { + return cs.Steps +} + +func (cs *LuetCompilationSpec) GetSeedImage() string { + return cs.Seed +} + +func (cs *LuetCompilationSpec) GetImage() string { + return cs.Image +} + +func (cs *LuetCompilationSpec) SetImage(s string) { + cs.Image = s +} + +func (cs *LuetCompilationSpec) SetSeedImage(s string) { + cs.Seed = s +} + +// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within +func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) { + spec := ` +FROM ` + cs.GetSeedImage() + ` +COPY . /luetbuild +WORKDIR /luetbuild +` + return spec, nil +} + +// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within +func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) { + spec := ` +FROM ` + image + for _, s := range cs.BuildSteps() { + spec = spec + ` +RUN ` + s + } + + return spec, nil +} + +func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error { + data, err := cs.RenderBuildImage() + if err != nil { + return err + } + return ioutil.WriteFile(path, []byte(data), 0644) +} + +func (cs *LuetCompilationSpec) WriteStepImageDefinition(fromimage, path string) error { + data, err := cs.RenderStepImage(fromimage) + if err != nil { + return err + } + return ioutil.WriteFile(path, []byte(data), 0644) } diff --git a/pkg/compiler/spec_test.go b/pkg/compiler/spec_test.go index 9aca79fb..20d2d955 100644 --- a/pkg/compiler/spec_test.go +++ b/pkg/compiler/spec_test.go @@ -17,10 +17,14 @@ package compiler_test import ( . "github.com/mudler/luet/pkg/compiler" + helpers "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/tree" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "io/ioutil" + "os" + "path/filepath" ) var _ = Describe("Spec", func() { @@ -43,6 +47,30 @@ var _ = Describe("Spec", func() { Expect(lspec.Steps).To(Equal([]string{"echo foo", "bar"})) Expect(lspec.Image).To(Equal("luet/base")) + Expect(lspec.Seed).To(Equal("luet/baseimage")) + tmpdir, err := ioutil.TempDir("", "tree") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpdir) // clean up + + err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile")) + Expect(err).ToNot(HaveOccurred()) + dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile")) + Expect(err).ToNot(HaveOccurred()) + Expect(dockerfile).To(Equal(` +FROM luet/baseimage +COPY . /luetbuild +WORKDIR /luetbuild +`)) + + err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile")) + Expect(err).ToNot(HaveOccurred()) + dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile")) + Expect(err).ToNot(HaveOccurred()) + Expect(dockerfile).To(Equal(` +FROM luet/base +RUN echo foo +RUN bar`)) + }) }) diff --git a/pkg/helpers/file.go b/pkg/helpers/file.go index c1313e46..943012f5 100644 --- a/pkg/helpers/file.go +++ b/pkg/helpers/file.go @@ -15,7 +15,10 @@ package helpers -import "os" +import ( + "io/ioutil" + "os" +) // Exists reports whether the named file or directory exists. func Exists(name string) bool { @@ -26,3 +29,11 @@ func Exists(name string) bool { } return true } + +func Read(file string) (string, error) { + dat, err := ioutil.ReadFile(file) + if err != nil { + return "", err + } + return string(dat), nil +} diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go new file mode 100644 index 00000000..437a8ebd --- /dev/null +++ b/pkg/tree/compiler_recipe.go @@ -0,0 +1,105 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +// Recipe is a builder imeplementation. + +// It reads a Tree and spit it in human readable form (YAML), called recipe, +// It also loads a tree (recipe) from a YAML (to a db, e.g. BoltDB), allowing to query it +// with the solver, using the package object. +package tree + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/mudler/luet/pkg/helpers" + pkg "github.com/mudler/luet/pkg/package" + "github.com/pkg/errors" +) + +const ( + CompilerDefinitionFile = "build.yaml" +) + +func NewCompilerRecipe() Builder { return &CompilerRecipe{} } + +// Recipe is the "general" reciper for Trees +type CompilerRecipe struct { + Recipe +} + +func (r *CompilerRecipe) Load(path string) error { + + if r.Tree() == nil { + r.PackageTree = NewDefaultTree() + } + + tmpfile, err := ioutil.TempFile("", "luet") + if err != nil { + return err + } + + r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name())) + // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() + + // the function that handles each file or dir + var ff = func(currentpath string, info os.FileInfo, err error) error { + + if info.Name() != DefinitionFile { + return nil // Skip with no errors + } + + dat, err := ioutil.ReadFile(currentpath) + if err != nil { + return errors.Wrap(err, "Error reading file "+currentpath) + } + pack, err := pkg.DefaultPackageFromYaml(dat) + if err != nil { + return errors.Wrap(err, "Error reading yaml "+currentpath) + } + + // Path is set only internally when tree is loaded from disk + pack.SetPath(filepath.Dir(currentpath)) + + // Instead of rdeps, have a different tree for build deps. + compileDefPath := pack.Rel(CompilerDefinitionFile) + if helpers.Exists(compileDefPath) { + dat, err = ioutil.ReadFile(currentpath) + if err != nil { + return errors.Wrap(err, "Error reading file "+currentpath) + } + packbuild, err := pkg.DefaultPackageFromYaml(dat) + if err != nil { + return errors.Wrap(err, "Error reading yaml "+currentpath) + } + pack.Requires(packbuild.GetRequires()) + pack.Conflicts(packbuild.GetConflicts()) + } + + _, err = r.Tree().GetPackageSet().CreatePackage(&pack) + if err != nil { + return errors.Wrap(err, "Error creating package "+pack.GetName()) + } + + return nil + } + + err = filepath.Walk(path, ff) + if err != nil { + return err + } + return nil +} diff --git a/tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml b/tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml index 88556826..4d3fedf4 100644 --- a/tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml +++ b/tests/fixtures/buildtree/app-admin/enman/1.4.0/build.yaml @@ -1,4 +1,5 @@ image: "luet/base" +seed: "luet/baseimage" steps: - echo foo - bar \ No newline at end of file