diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 737e191b..acc5fd79 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -18,6 +18,8 @@ package compiler import ( "errors" "io/ioutil" + "os" + "path/filepath" "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" @@ -41,7 +43,7 @@ func NewLuetCompiler(backend CompilerBackend, t pkg.Tree) Compiler { } } -func (cs *LuetCompiler) Compile(p CompilationSpec) (Artifact, error) { +func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based // on how we compute the resolvable tree. @@ -55,42 +57,74 @@ func (cs *LuetCompiler) Compile(p CompilationSpec) (Artifact, error) { p.SetSeedImage(p.GetImage()) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, // and we build all the images first. + buildDir := p.Rel("build") + + // First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow) + err := helpers.CopyDir(p.GetPackage().GetPath(), buildDir) + if err != nil { + return nil, err + + } + // First we create the builder image - p.WriteBuildImageDefinition(p.Rel(p.GetPackage().GetFingerPrint() + "-builder.dockerfile")) + p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")) builderOpts := CompilerBackendOptions{ ImageName: "luet/" + p.GetPackage().GetFingerPrint() + "-builder", - SourcePath: p.GetOutputPath(), - DockerFileName: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.dockerfile"), - Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.rootfs.tar"), + SourcePath: buildDir, + DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile", + Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"), } - err := cs.Backend.BuildImage(builderOpts) + err = cs.Backend.BuildImage(builderOpts) + if err != nil { + return nil, err + } + + err = cs.Backend.ExportImage(builderOpts) if err != nil { return nil, err } // Then we write the step image, which uses the builder one - p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", p.Rel(p.GetPackage().GetFingerPrint()+".dockerfile")) + p.WriteStepImageDefinition("luet/"+p.GetPackage().GetFingerPrint()+"-builder", filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")) runnerOpts := CompilerBackendOptions{ ImageName: "luet/" + p.GetPackage().GetFingerPrint(), - SourcePath: p.GetOutputPath(), - DockerFileName: p.Rel(p.GetPackage().GetFingerPrint() + ".dockerfile"), - Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".rootfs.tar"), + SourcePath: buildDir, + DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile", + Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), } err = cs.Backend.ImageDefinitionToTar(runnerOpts) if err != nil { return nil, err } + diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar")) + if err != nil { + return nil, err + } + // TODO: Handle caching and optionally do not remove things err = cs.Backend.RemoveImage(builderOpts) if err != nil { return nil, err } - // TODO: Delta should be the artifact - return NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".rootfs.tar")), nil + rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") + defer os.RemoveAll(rootfs) // clean up + + // TODO: Compression and such + err = cs.Backend.ExtractRootfs(CompilerBackendOptions{SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions) + if err != nil { + return nil, err + } + artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions) + if err != nil { + return nil, err + } + + return artifact, nil } + // TODO: Solve - hash return nil, errors.New("Not implemented yet") } @@ -100,6 +134,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { if err != nil { return nil, err } + buildFile := pack.Rel(BuildFile) if !helpers.Exists(buildFile) { return nil, errors.New("No build file present for " + p.GetFingerPrint()) @@ -109,7 +144,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { if err != nil { return nil, err } - return NewLuetCompilationSpec(dat, p) + return NewLuetCompilationSpec(dat, pack) } func (cs *LuetCompiler) GetBackend() CompilerBackend { diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go index 5b55c5c4..2512d058 100644 --- a/pkg/compiler/interface.go +++ b/pkg/compiler/interface.go @@ -20,7 +20,7 @@ import ( ) type Compiler interface { - Compile(CompilationSpec) (Artifact, error) + Compile(int, bool, CompilationSpec) (Artifact, error) FromPackage(pkg.Package) (CompilationSpec, error) SetBackend(CompilerBackend) diff --git a/pkg/helpers/archive.go b/pkg/helpers/archive.go new file mode 100644 index 00000000..4c545933 --- /dev/null +++ b/pkg/helpers/archive.go @@ -0,0 +1,62 @@ +// Copyright © 2019 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package helpers + +import ( + "io" + "os" + + "github.com/docker/docker/pkg/archive" +) + +func Tar(src, dest string) error { + out, err := os.Create(dest) + if err != nil { + return err + } + defer out.Close() + + fs, err := archive.Tar(src, archive.Uncompressed) + if err != nil { + return err + } + defer fs.Close() + + _, err = io.Copy(out, fs) + if err != nil { + return err + } + + err = out.Sync() + if err != nil { + return err + } + return err +} + +// Untar just a wrapper around the docker functions +func Untar(src, dest string, sameOwner bool) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + return archive.Untar(in, dest, &archive.TarOptions{ + NoLchown: !sameOwner, + ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted' + }) +} diff --git a/pkg/helpers/file.go b/pkg/helpers/file.go index 943012f5..ea065b1d 100644 --- a/pkg/helpers/file.go +++ b/pkg/helpers/file.go @@ -16,8 +16,11 @@ package helpers import ( + "fmt" + "io" "io/ioutil" "os" + "path/filepath" ) // Exists reports whether the named file or directory exists. @@ -37,3 +40,113 @@ func Read(file string) (string, error) { } return string(dat), nil } + +// CopyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +func IsDirectory(path string) (bool, error) { + fileInfo, err := os.Stat(path) + if err != nil { + return false, err + } + return fileInfo.IsDir(), err +} + +// CopyDir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +// Symlinks are ignored and skipped. +func CopyDir(src string, dst string) (err error) { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf("source is not a directory") + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf("destination already exists") + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := ioutil.ReadDir(src) + if err != nil { + return + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = CopyDir(srcPath, dstPath) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + err = CopyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return +} diff --git a/pkg/package/package.go b/pkg/package/package.go index fd7146db..3f82f14d 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -105,7 +105,7 @@ type DefaultPackage struct { // TODO: Annotations? // Path is set only internally when tree is loaded from disk - Path string `json:"path,omitempty"` // primary key with auto increment + Path string `json:"path,omitempty"` } // State represent the package state diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index 437a8ebd..6294e5bd 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -47,12 +47,13 @@ func (r *CompilerRecipe) Load(path string) error { r.PackageTree = NewDefaultTree() } - tmpfile, err := ioutil.TempFile("", "luet") - if err != nil { - return err - } + //tmpfile, err := ioutil.TempFile("", "luet") + //if err != nil { + // return err + //} - r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name())) + r.Tree().SetPackageSet(pkg.NewInMemoryDatabase(false)) + //r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name())) // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() // the function that handles each file or dir @@ -97,7 +98,7 @@ func (r *CompilerRecipe) Load(path string) error { return nil } - err = filepath.Walk(path, ff) + err := filepath.Walk(path, ff) if err != nil { return err }