diff --git a/cmd/build.go b/cmd/build.go index 4be9372a..88940802 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -16,7 +16,6 @@ package cmd import ( "fmt" - "io/ioutil" "os" "path/filepath" @@ -25,6 +24,7 @@ import ( "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/compiler/types/artifact" compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + "github.com/mudler/luet/pkg/installer" "github.com/mudler/luet/pkg/compiler/types/compression" "github.com/mudler/luet/pkg/compiler/types/options" @@ -44,17 +44,17 @@ var buildCmd = &cobra.Command{ Long: `Builds one or more packages from a tree (current directory is implied): $ luet build utils/busybox utils/yq ... - + Builds all packages - + $ luet build --all - + Builds only the leaf packages: - + $ luet build --full Build package revdeps: - + $ luet build --revdeps utils/yq Build package without dependencies (needs the images already in the host, or either need to be available online): @@ -69,7 +69,6 @@ Build packages specifying multiple definition trees: viper.BindPFlag("destination", cmd.Flags().Lookup("destination")) viper.BindPFlag("backend", cmd.Flags().Lookup("backend")) viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged")) - viper.BindPFlag("database", cmd.Flags().Lookup("database")) viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps")) viper.BindPFlag("all", cmd.Flags().Lookup("all")) viper.BindPFlag("compression", cmd.Flags().Lookup("compression")) @@ -101,7 +100,6 @@ Build packages specifying multiple definition trees: privileged := viper.GetBool("privileged") revdeps := viper.GetBool("revdeps") all := viper.GetBool("all") - databaseType := viper.GetString("database") compressionType := viper.GetString("compression") imageRepository := viper.GetString("image-repository") values := viper.GetStringSlice("values") @@ -122,26 +120,25 @@ Build packages specifying multiple definition trees: LuetCfg.GetLogging().SetLogLevel("error") } pretend, _ := cmd.Flags().GetBool("pretend") + fromRepo, _ := cmd.Flags().GetBool("from-repositories") + compilerSpecs := compilerspec.NewLuetCompilationspecs() var db pkg.PackageDatabase compilerBackend, err := compiler.NewBackend(backendType) helpers.CheckErr(err) - switch databaseType { - case "memory": - db = pkg.NewInMemoryDatabase(false) - - case "boltdb": - tmpdir, err := ioutil.TempDir("", "package") - helpers.CheckErr(err) - db = pkg.NewBoltDatabase(tmpdir) - - } + db = pkg.NewInMemoryDatabase(false) defer db.Clean() generalRecipe := tree.NewCompilerRecipe(db) + if fromRepo { + if err := installer.LoadBuildTree(generalRecipe, db, LuetCfg); err != nil { + Warning("errors while loading trees from repositories", err.Error()) + } + } + for _, src := range treePaths { Info("Loading tree", src) helpers.CheckErr(generalRecipe.Load(src)) @@ -202,7 +199,6 @@ Build packages specifying multiple definition trees: } } else if !all { for _, a := range args { - pack, err := helpers.ParsePackageStr(a) if err != nil { Fatal("Invalid package string ", a, ": ", err.Error()) @@ -310,7 +306,6 @@ func init() { buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.") buildCmd.Flags().String("backend", "docker", "backend used (docker,img)") buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)") - buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)") buildCmd.Flags().Bool("revdeps", false, "Build with revdeps") buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree") buildCmd.Flags().Bool("full", false, "Build all packages (optimized)") @@ -333,6 +328,7 @@ func init() { buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts") buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)") buildCmd.Flags().Bool("live-output", LuetCfg.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.") + buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from") buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled") buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from") diff --git a/cmd/install.go b/cmd/install.go index 9d8500d7..6aad2be7 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -70,16 +70,6 @@ To force install a package: toInstall = append(toInstall, pack) } - // This shouldn't be necessary, but we need to unmarshal the repositories to a concrete struct, thus we need to port them back to the Repositories type - repos := installer.Repositories{} - for _, repo := range LuetCfg.SystemRepositories { - if !repo.Enable { - continue - } - r := installer.NewSystemRepository(repo) - repos = append(repos, r) - } - stype := LuetCfg.Viper.GetString("solver.type") discount := LuetCfg.Viper.GetFloat64("solver.discount") rate := LuetCfg.Viper.GetFloat64("solver.rate") @@ -111,6 +101,7 @@ To force install a package: } Debug("Solver", LuetCfg.GetSolverOptions().CompactString()) + repos := installer.SystemRepositories(LuetCfg) // Load config protect configs installer.LoadConfigProtectConfs(LuetCfg) diff --git a/go.mod b/go.mod index cb2d53d5..e1cf7fa3 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,10 @@ require ( github.com/genuinetools/img v0.5.11 github.com/ghodss/yaml v1.0.0 github.com/google/go-containerregistry v0.2.1 + github.com/google/renameio v1.0.0 github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/go-version v1.2.1 + github.com/imdario/mergo v0.3.8 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/jedib0t/go-pretty/v6 v6.0.5 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 @@ -51,7 +53,7 @@ require ( github.com/theupdateframework/notary v0.7.0 go.etcd.io/bbolt v1.3.5 go.uber.org/atomic v1.5.1 // indirect - go.uber.org/multierr v1.4.0 // indirect + go.uber.org/multierr v1.4.0 go.uber.org/zap v1.13.0 google.golang.org/grpc v1.29.1 gopkg.in/yaml.v2 v2.3.0 diff --git a/go.sum b/go.sum index e532987e..c0cd5bde 100644 --- a/go.sum +++ b/go.sum @@ -493,6 +493,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/renameio v1.0.0 h1:xhp2CnJmgQmpJU4RY8chagahUq5mbPPAbiSQstKpVMA= +github.com/google/renameio v1.0.0/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 h1:JM174NTeGNJ2m/oLH3UOWOvWQQKd+BoL3hcSCUWFLt0= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index a51cd6f2..02f1dcec 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -27,6 +27,7 @@ import ( "sync" "time" + "github.com/imdario/mergo" bus "github.com/mudler/luet/pkg/bus" "github.com/mudler/luet/pkg/compiler/backend" artifact "github.com/mudler/luet/pkg/compiler/types/artifact" @@ -37,6 +38,7 @@ import ( pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/solver" "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) const BuildFile = "build.yaml" @@ -48,19 +50,11 @@ type ArtifactIndex []*artifact.PackageArtifact func (i ArtifactIndex) CleanPath() ArtifactIndex { newIndex := ArtifactIndex{} for _, art := range i { - // FIXME: This is a dup and makes difficult to add attributes to artifacts - newIndex = append(newIndex, &artifact.PackageArtifact{ - Path: path.Base(art.Path), - SourceAssertion: art.SourceAssertion, - CompileSpec: art.CompileSpec, - Dependencies: art.Dependencies, - CompressionType: art.CompressionType, - Checksums: art.Checksums, - Files: art.Files, - }) + copy := art.ShallowCopy() + copy.Path = path.Base(art.Path) + newIndex = append(newIndex, copy) } return newIndex - //Update if exists, otherwise just create } type LuetCompiler struct { @@ -544,7 +538,7 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string { } func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { - metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml" + metaFile := spec.GetPackage().GetMetadataFilePath() dat, err := ioutil.ReadFile(spec.Rel(metaFile)) if err != nil { return nil, errors.Wrap(err, "Error reading file "+metaFile) @@ -731,6 +725,12 @@ func genImageList(refs []string, hash string) []string { } func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { + if len(p.BuildOptions.PullImageRepository) != 0 { + orig := cs.Options.PullImageRepository + cs.Options.PullImageRepository = append(orig, p.BuildOptions.PullImageRepository...) + defer func() { cs.Options.PullImageRepository = orig }() + } + Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:") Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage())) @@ -866,7 +866,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil type templatedata map[string]interface{} -func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { +func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package) ([]byte, error) { var dataresult []byte val := pack.Rel(DefinitionFile) @@ -876,7 +876,7 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { if err != nil { return nil, errors.Wrap(err, "unmarshalling values") } - cs.Options.BuildValues = []map[string]interface{}{(map[string]interface{})(dst)} + cs.Options.BuildValues = append(vals, (map[string]interface{})(dst)) if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil { val = pack.Rel(CollectionFile) @@ -897,14 +897,46 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { } raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion()) + td := templatedata{} + if len(vals) > 0 { + for _, bv := range vals { + current := templatedata(bv) + if err := mergo.Merge(&td, current); err != nil { + return nil, errors.Wrap(err, "merging values maps") + } + } + } - dat, err := helpers.RenderHelm(string(dataBuild), raw, dst) + if err := mergo.Merge(&td, raw); err != nil { + return nil, errors.Wrap(err, "merging values maps") + } + + dat, err := helpers.RenderHelm(string(dataBuild), td, dst) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } dataresult = []byte(dat) } else { - out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile...) + bv := cs.Options.BuildValuesFile + if len(vals) > 0 { + valuesdir, err := ioutil.TempDir("", "genvalues") + if err != nil { + return nil, errors.Wrap(err, "Could not create tempdir") + } + defer os.RemoveAll(valuesdir) // clean up + for _, b := range vals { + out, err := yaml.Marshal(b) + if err != nil { + return nil, errors.Wrap(err, "while marshalling values file") + } + f := filepath.Join(valuesdir, helpers.RandStringRunes(20)) + if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil { + return nil, errors.Wrap(err, "while writing temporary values file") + } + bv = append([]string{f}, bv...) + } + } + out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } @@ -922,12 +954,43 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilatio return nil, err } - bytes, err := cs.templatePackage(pack) + opts := options.Compiler{} + + artifactMetadataFile := filepath.Join(p.GetTreeDir(), p.GetMetadataFilePath()) + + if fi, err := os.Stat(artifactMetadataFile); err == nil { + f, err := os.Open(fi.Name()) + if err != nil { + return nil, errors.Wrapf(err, "could not open %s", fi.Name()) + } + dat, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + art, err := artifact.NewPackageArtifactFromYaml(dat) + if err != nil { + return nil, errors.Wrap(err, "could not decode package from yaml") + } + + opts = art.CompileSpec.BuildOptions + opts.PushImageRepository = "" + + } else if !os.IsNotExist(err) { + Debug("error reading already existing artifact metadata file: ", err.Error()) + } + + bytes, err := cs.templatePackage(opts.BuildValues, pack) if err != nil { return nil, errors.Wrap(err, "while rendering package template") } - return compilerspec.NewLuetCompilationSpec(bytes, pack) + newSpec, err := compilerspec.NewLuetCompilationSpec(bytes, pack) + if err != nil { + return nil, err + } + newSpec.BuildOptions = opts + + return newSpec, err } // GetBackend returns the current compilation backend diff --git a/pkg/compiler/types/artifact/artifact.go b/pkg/compiler/types/artifact/artifact.go index ed0506dd..b9e27ace 100644 --- a/pkg/compiler/types/artifact/artifact.go +++ b/pkg/compiler/types/artifact/artifact.go @@ -62,6 +62,11 @@ type PackageArtifact struct { Files []string `json:"files"` } +func (p *PackageArtifact) ShallowCopy() *PackageArtifact { + copy := *p + return © +} + func NewPackageArtifact(path string) *PackageArtifact { return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None} } @@ -129,7 +134,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error { return errors.Wrap(err, "While marshalling for PackageArtifact YAML") } - err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm) + err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm) if err != nil { return errors.Wrap(err, "While writing PackageArtifact YAML") } diff --git a/pkg/helpers/file.go b/pkg/helpers/file.go index 0cdf1d6a..e866679e 100644 --- a/pkg/helpers/file.go +++ b/pkg/helpers/file.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "io/ioutil" + "math/rand" "os" "path/filepath" "sort" @@ -26,10 +27,41 @@ import ( "syscall" "time" + "github.com/google/renameio" copy "github.com/otiai10/copy" "github.com/pkg/errors" ) +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func RandStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func Move(src, dst string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + t, err := renameio.TempFile("", dst) + if err != nil { + return err + } + defer t.Cleanup() + + _, err = io.Copy(t, f) + if err != nil { + return err + } + return t.CloseAtomicallyReplace() +} + func OrderFiles(target string, files []string) ([]string, []string) { var newFiles []string diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 9f4d48f4..ab4dfa70 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -16,6 +16,7 @@ package installer import ( + "fmt" "io/ioutil" "os" "path" @@ -27,6 +28,7 @@ import ( artifact "github.com/mudler/luet/pkg/compiler/types/artifact" compression "github.com/mudler/luet/pkg/compiler/types/compression" + "go.uber.org/multierr" "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" @@ -100,6 +102,40 @@ func NewLuetSystemRepositoryMetadata(file string, removeFile bool) (*LuetSystemR return ans, nil } +// SystemRepositories returns the repositories from the local configuration file +func SystemRepositories(c *config.LuetConfig) Repositories { + repos := Repositories{} + for _, repo := range c.SystemRepositories { + if !repo.Enable { + continue + } + r := NewSystemRepository(repo) + repos = append(repos, r) + } + return repos +} + +// LoadBuildTree loads to the tree the compilation specs from the system repositories +func LoadBuildTree(t tree.Builder, db pkg.PackageDatabase, c *config.LuetConfig) error { + var reserr error + repos := SystemRepositories(c) + for _, r := range repos { + repodir, err := config.LuetCfg.GetSystem().TempDir(r.Name) + if err != nil { + reserr = multierr.Append(reserr, err) + } + if err := r.SyncBuildMetadata(repodir); err != nil { + reserr = multierr.Append(reserr, err) + } + if err := t.Load(filepath.Join(repodir, "tree")); err != nil { + reserr = multierr.Append(reserr, err) + } + } + repos.SyncDatabase(db) + + return reserr +} + func (m *LuetSystemRepositoryMetadata) WriteFile(path string) error { data, err := yaml.Marshal(m) if err != nil { @@ -471,7 +507,7 @@ func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot st treeFile, err := r.GetRepositoryFile(fileKey) if err != nil { treeFile = defaults - r.SetRepositoryFile(fileKey, treeFile) + // r.SetRepositoryFile(fileKey, treeFile) } a := artifact.NewPackageArtifact(filepath.Join(repositoryRoot, treeFile.GetFileName())) @@ -594,6 +630,70 @@ func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (*artifact.PackageA return nil, errors.New("Not found") } +func (r *LuetSystemRepository) getRepoFile(c Client, key string) (*artifact.PackageArtifact, error) { + + treeFile, err := r.GetRepositoryFile(key) + if err != nil { + return nil, errors.Wrapf(err, "key %s not present in the repository", key) + } + + // Get Tree + downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName()) + if err != nil { + return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName()) + } + //defer os.Remove(downloadedTreeFile) + + treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile) + treeFileArtifact.Checksums = treeFile.GetChecksums() + treeFileArtifact.CompressionType = treeFile.GetCompressionType() + + err = treeFileArtifact.Verify() + if err != nil { + return nil, errors.Wrap(err, "file integrity check failure") + } + + return treeFileArtifact, nil + +} + +func (r *LuetSystemRepository) SyncBuildMetadata(path string) error { + + repo, err := r.Sync(false) + if err != nil { + return errors.Wrap(err, "while syncronizing repository") + } + + c := repo.Client() + if c == nil { + return errors.New("no client could be generated from repository") + } + + a, err := repo.getRepoFile(c, REPOFILE_COMPILER_TREE_KEY) + if err != nil { + return fmt.Errorf("failed while getting: %s", REPOFILE_COMPILER_TREE_KEY) + } + + defer os.RemoveAll(a.Path) + + if err := a.Unpack(filepath.Join(path, "tree"), false); err != nil { + return errors.Wrapf(err, "while unpacking: %s", REPOFILE_COMPILER_TREE_KEY) + } + + for _, ai := range repo.GetTree().GetDatabase().World() { + // Retrieve remote repository.yaml for retrieve revision and date + file, err := c.DownloadFile(ai.GetMetadataFilePath()) + if err != nil { + return errors.Wrapf(err, "while downloading metadata for %s", ai.HumanReadableString()) + } + if err := helpers.Move(file, filepath.Join(path, ai.GetMetadataFilePath())); err != nil { + return err + } + } + + return nil +} + func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { var repoUpdated bool = false var treefs, metafs string @@ -612,7 +712,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { } repobasedir := config.LuetCfg.GetSystem().GetRepoDatabaseDirPath(r.GetName()) - repo, err := r.ReadSpecFile(file) + downloadedRepoMeta, err := r.ReadSpecFile(file) if err != nil { return nil, err } @@ -624,8 +724,8 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { if !force { localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE)) if localRepo != nil { - if localRepo.GetRevision() == repo.GetRevision() && - localRepo.GetLastUpdate() == repo.GetLastUpdate() { + if localRepo.GetRevision() == downloadedRepoMeta.GetRevision() && + localRepo.GetLastUpdate() == downloadedRepoMeta.GetLastUpdate() { repoUpdated = true } } @@ -652,47 +752,22 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { } } - // POST: treeFile and metaFile are present. I check this inside - // ReadSpecFile and NewLuetSystemRepositoryFromYaml - treeFile, _ := repo.GetRepositoryFile(REPOFILE_TREE_KEY) - metaFile, _ := repo.GetRepositoryFile(REPOFILE_META_KEY) - + // treeFile and metaFile must be present, they aren't optional if !repoUpdated { - // Get Tree - downloadedTreeFile, err := c.DownloadFile(treeFile.GetFileName()) + treeFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_TREE_KEY) if err != nil { - return nil, errors.Wrap(err, "While downloading "+treeFile.GetFileName()) - } - defer os.Remove(downloadedTreeFile) - - // Treat the file as artifact, in order to verify it - treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile) - treeFileArtifact.Checksums = treeFile.GetChecksums() - treeFileArtifact.CompressionType = treeFile.GetCompressionType() - - err = treeFileArtifact.Verify() - if err != nil { - return nil, errors.Wrap(err, "Tree integrity check failure") + return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_TREE_KEY) } + defer os.Remove(treeFileArtifact.Path) Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.") - // Get Repository Metadata - downloadedMeta, err := c.DownloadFile(metaFile.GetFileName()) + metaFileArtifact, err := downloadedRepoMeta.getRepoFile(c, REPOFILE_META_KEY) if err != nil { - return nil, errors.Wrap(err, "While downloading "+metaFile.GetFileName()) - } - defer os.Remove(downloadedMeta) - - metaFileArtifact := artifact.NewPackageArtifact(downloadedMeta) - metaFileArtifact.Checksums = metaFile.GetChecksums() - metaFileArtifact.CompressionType = metaFile.GetCompressionType() - - err = metaFileArtifact.Verify() - if err != nil { - return nil, errors.Wrap(err, "Metadata integrity check failure") + return nil, errors.Wrapf(err, "while fetching '%s'", REPOFILE_META_KEY) } + defer os.Remove(metaFileArtifact.Path) Debug("Metadata tarball for the repository " + r.GetName() + " downloaded correctly.") @@ -722,17 +797,17 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { return nil, errors.Wrap(err, "Error met while unpacking metadata") } - tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64) + tsec, _ := strconv.ParseInt(downloadedRepoMeta.GetLastUpdate(), 10, 64) InfoC( aurora.Bold( - aurora.Red(":house: Repository "+repo.GetName()+" revision: ")).String() + - aurora.Bold(aurora.Green(repo.GetRevision())).String() + " - " + + aurora.Red(":house: Repository "+downloadedRepoMeta.GetName()+" revision: ")).String() + + aurora.Bold(aurora.Green(downloadedRepoMeta.GetRevision())).String() + " - " + aurora.Bold(aurora.Green(time.Unix(tsec, 0).String())).String(), ) } else { - Info("Repository", repo.GetName(), "is already up to date.") + Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.") } meta, err := NewLuetSystemRepositoryMetadata( @@ -741,7 +816,7 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { if err != nil { return nil, errors.Wrap(err, "While processing "+REPOSITORY_METAFILE) } - repo.SetIndex(meta.ToArtifactIndex()) + downloadedRepoMeta.SetIndex(meta.ToArtifactIndex()) reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false)) err = reciper.Load(treefs) @@ -749,29 +824,33 @@ func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { return nil, errors.Wrap(err, "Error met while unpacking rootfs") } - repo.SetTree(reciper) - repo.SetTreePath(treefs) + downloadedRepoMeta.SetTree(reciper) + downloadedRepoMeta.SetTreePath(treefs) // Copy the local available data to the one which was synced // e.g. locally we can override the type (disk), or priority // while remotely it could be advertized differently - repo.SetUrls(r.GetUrls()) - repo.SetAuthentication(r.GetAuthentication()) - repo.SetType(r.GetType()) - repo.SetPriority(r.GetPriority()) - repo.SetName(r.GetName()) - repo.SetVerify(r.GetVerify()) + r.fill(downloadedRepoMeta) InfoC( aurora.Yellow(":information_source:").String() + aurora.Magenta("Repository: ").String() + - aurora.Green(aurora.Bold(repo.GetName()).String()).String() + + aurora.Green(aurora.Bold(downloadedRepoMeta.GetName()).String()).String() + aurora.Magenta(" Priority: ").String() + - aurora.Bold(aurora.Green(repo.GetPriority())).String() + + aurora.Bold(aurora.Green(downloadedRepoMeta.GetPriority())).String() + aurora.Magenta(" Type: ").String() + - aurora.Bold(aurora.Green(repo.GetType())).String(), + aurora.Bold(aurora.Green(downloadedRepoMeta.GetType())).String(), ) - return repo, nil + return downloadedRepoMeta, nil +} + +func (r *LuetSystemRepository) fill(r2 *LuetSystemRepository) { + r2.SetUrls(r.GetUrls()) + r2.SetAuthentication(r.GetAuthentication()) + r2.SetType(r.GetType()) + r2.SetPriority(r.GetPriority()) + r2.SetName(r.GetName()) + r2.SetVerify(r.GetVerify()) } func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepository) { diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 131f1d0b..8d0b8a06 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -44,23 +44,22 @@ type dockerRepositoryGenerator struct { } func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) { - return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force) -} -func pushImage(b compiler.CompilerBackend, image string, force bool) error { - if b.ImageAvailable(image) && !force { - Debug("Image", image, "already present, skipping") - return nil - } - return b.Push(backend.Options{ImageName: image}) -} - -func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]*artifact.PackageArtifact, error) { - Info("Generating docker images for packages in", imagePrefix) + Info("Generating docker images for packages in", l.imagePrefix) var art []*artifact.PackageArtifact var ff = func(currentpath string, info os.FileInfo, err error) error { + if err != nil { + Debug("Skipping", info.Name(), err.Error()) + return nil + } - if !strings.HasSuffix(info.Name(), ".metadata.yaml") { + if strings.HasSuffix(info.Name(), ".metadata.yaml") { + a := artifact.NewPackageArtifact(info.Name()) + imageRepo := fmt.Sprintf("%s:%s", l.imagePrefix, filepath.Base(info.Name())) + + if err := l.pushFileFromArtifact(a, imageRepo); err != nil { + return errors.Wrap(err, "while pushing file from artifact") + } return nil // Skip with no errors } @@ -85,19 +84,19 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, return nil } - packageImage := fmt.Sprintf("%s:%s", imagePrefix, a.CompileSpec.GetPackage().ImageID()) + packageImage := fmt.Sprintf("%s:%s", l.imagePrefix, a.CompileSpec.GetPackage().ImageID()) - if imagePush && b.ImageAvailable(packageImage) && !force { + if l.imagePush && l.b.ImageAvailable(packageImage) && !l.force { Info("Image", packageImage, "already present, skipping. use --force-push to override") } else { Info("Generating final image", packageImage, "for package ", a.CompileSpec.GetPackage().HumanReadableString()) - if opts, err := a.GenerateFinalImage(packageImage, b, true); err != nil { + if opts, err := a.GenerateFinalImage(packageImage, l.b, true); err != nil { return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) } } - if imagePush { - if err := pushImage(b, packageImage, force); err != nil { + if l.imagePush { + if err := pushImage(l.b, packageImage, l.force); err != nil { return errors.Wrapf(err, "Failed while pushing image: '%s'", packageImage) } } @@ -115,13 +114,21 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, return art, nil } -func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string, r *LuetSystemRepository) error { +func pushImage(b compiler.CompilerBackend, image string, force bool) error { + if b.ImageAvailable(image) && !force { + Debug("Image", image, "already present, skipping") + return nil + } + return b.Push(backend.Options{ImageName: image}) +} + +func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string) error { Debug("Generating image", imageTree) - if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil { + if opts, err := a.GenerateFinalImage(imageTree, d.b, false); err != nil { return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) } - if r.PushImages { - if err := pushImage(r.GetBackend(), imageTree, true); err != nil { + if d.imagePush { + if err := pushImage(d.b, imageTree, true); err != nil { return errors.Wrapf(err, "Failed while pushing image: '%s'", imageTree) } } @@ -144,13 +151,13 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys a := artifact.NewPackageArtifact(tempRepoFile) imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE) - if err := d.pushFileFromArtifact(a, imageRepo, r); err != nil { + if err := d.pushFileFromArtifact(a, imageRepo); err != nil { return errors.Wrap(err, "while pushing file from artifact") } return nil } -func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, r *LuetSystemRepository) error { +func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, b compiler.CompilerBackend) error { // we generate a new archive containing the required compressed file. // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file treeArchive, err := artifact.CreateArtifactForFile(a.Path) @@ -159,7 +166,7 @@ func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArt } imageTree := fmt.Sprintf("%s:%s", d.imagePrefix, a.GetFileName()) - return d.pushFileFromArtifact(treeArchive, imageTree, r) + return d.pushFileFromArtifact(treeArchive, imageTree) } // Generate creates a Docker luet repository @@ -216,7 +223,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi // we generate a new archive containing the required compressed file. // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file - if err := d.pushImageFromArtifact(a, r); err != nil { + if err := d.pushImageFromArtifact(a, d.b); err != nil { return errors.Wrap(err, "error met while pushing runtime tree") } @@ -226,7 +233,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi } // we generate a new archive containing the required compressed file. // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file - if err := d.pushImageFromArtifact(a, r); err != nil { + if err := d.pushImageFromArtifact(a, d.b); err != nil { return errors.Wrap(err, "error met while pushing compiler tree") } @@ -242,7 +249,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi return errors.Wrap(err, "failed adding Metadata file to repository") } - if err := d.pushImageFromArtifact(a, r); err != nil { + if err := d.pushImageFromArtifact(a, d.b); err != nil { return errors.Wrap(err, "error met while pushing docker image from artifact") } diff --git a/pkg/package/package.go b/pkg/package/package.go index 82f0f3a0..53cfa7ec 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -113,6 +113,10 @@ type Package interface { GetBuildTimestamp() string Clone() Package + + GetMetadataFilePath() string + SetTreeDir(s string) + GetTreeDir() string } type Tree interface { @@ -210,6 +214,11 @@ func (t *DefaultPackage) JSON() ([]byte, error) { return buffer.Bytes(), err } +// GetMetadataFilePath returns the canonical name of an artifact metadata file +func (d *DefaultPackage) GetMetadataFilePath() string { + return d.GetFingerPrint() + ".metadata.yaml" +} + // DefaultPackage represent a standard package definition type DefaultPackage struct { ID int `storm:"id,increment" json:"id"` // primary key with auto increment @@ -235,6 +244,8 @@ type DefaultPackage struct { BuildTimestamp string `json:"buildtimestamp,omitempty"` Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too. + + treeDir string } // State represent the package state @@ -251,6 +262,12 @@ func NewPackage(name, version string, requires []*DefaultPackage, conflicts []*D } } +func (p *DefaultPackage) SetTreeDir(s string) { + p.treeDir = s +} +func (p *DefaultPackage) GetTreeDir() string { + return p.treeDir +} func (p *DefaultPackage) String() string { b, err := p.JSON() if err != nil { diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index 3057d80a..7bd4bcb1 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -98,6 +98,7 @@ func (r *CompilerRecipe) Load(path string) error { } // Path is set only internally when tree is loaded from disk pack.SetPath(filepath.Dir(currentpath)) + pack.SetTreeDir(path) // Instead of rdeps, have a different tree for build deps. compileDefPath := pack.Rel(CompilerDefinitionFile) @@ -126,18 +127,25 @@ func (r *CompilerRecipe) Load(path string) error { } case CollectionFile: + dat, err := ioutil.ReadFile(currentpath) if err != nil { return errors.Wrap(err, "Error reading file "+currentpath) } + packs, err := pkg.DefaultPackagesFromYaml(dat) if err != nil { return errors.Wrap(err, "Error reading yaml "+currentpath) } + packsRaw, err := pkg.GetRawPackages(dat) + if err != nil { + return errors.Wrap(err, "Error reading raw packages from "+currentpath) + } for _, pack := range packs { pack.SetPath(filepath.Dir(currentpath)) + pack.SetTreeDir(path) // Instead of rdeps, have a different tree for build deps. compileDefPath := pack.Rel(CompilerDefinitionFile) @@ -170,9 +178,7 @@ func (r *CompilerRecipe) Load(path string) error { return errors.Wrap(err, "Error creating package "+pack.GetName()) } } - } - return nil }