From c9b684523f1dd0733d8aac751ad45b650f48d5e7 Mon Sep 17 00:00:00 2001 From: Daniele Rondina Date: Sun, 12 Jan 2020 23:31:43 +0100 Subject: [PATCH] installer: Add support for cached repository --- pkg/helpers/archive.go | 26 ++++++++-- pkg/installer/client/http.go | 12 ++--- pkg/installer/installer.go | 2 +- pkg/installer/interface.go | 2 +- pkg/installer/repository.go | 98 +++++++++++++++++++++++++++--------- 5 files changed, 104 insertions(+), 36 deletions(-) diff --git a/pkg/helpers/archive.go b/pkg/helpers/archive.go index 4c545933..9fff82b8 100644 --- a/pkg/helpers/archive.go +++ b/pkg/helpers/archive.go @@ -18,8 +18,11 @@ package helpers import ( "io" "os" + //"os/user" + //"strconv" "github.com/docker/docker/pkg/archive" + //"github.com/docker/docker/pkg/idtools" ) func Tar(src, dest string) error { @@ -55,8 +58,25 @@ func Untar(src, dest string, sameOwner bool) error { } defer in.Close() - return archive.Untar(in, dest, &archive.TarOptions{ - NoLchown: !sameOwner, + opts := &archive.TarOptions{ + // NOTE: NoLchown boolean is used for chmod of the symlink + // Probably it's needed set this always to true. + NoLchown: true, ExcludePatterns: []string{"dev/"}, // prevent 'operation not permitted' - }) + } + + /* + u, err := user.Current() + if err != nil { + return err + } + // TODO: This seems not sufficient for untar with normal user. + if u.Uid != "0" { + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(u.Gid) + opts.ChownOpts = &idtools.Identity{UID: uid, GID: gid} + } + */ + + return archive.Untar(in, dest, opts) } diff --git a/pkg/installer/client/http.go b/pkg/installer/client/http.go index c7cf630a..6971792e 100644 --- a/pkg/installer/client/http.go +++ b/pkg/installer/client/http.go @@ -49,15 +49,16 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti if err != nil { return nil, err } + defer os.RemoveAll(temp) + + file, err = ioutil.TempFile(temp, "HttpClient") + if err != nil { + return nil, err + } for _, uri := range c.RepoData.Urls { Info("Downloading artifact", artifactName, "from", uri) - file, err = ioutil.TempFile(temp, "HttpClient") - if err != nil { - continue - } - u, err = url.Parse(uri) if err != nil { continue @@ -103,7 +104,6 @@ func (c *HttpClient) DownloadFile(name string) (string, error) { if err != nil { continue } - //defer os.Remove(file.Name()) u, err = url.Parse(uri) if err != nil { continue diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go index 1c6f77c0..e867e352 100644 --- a/pkg/installer/installer.go +++ b/pkg/installer/installer.go @@ -128,7 +128,7 @@ func (l *LuetInstaller) SyncRepositories(inMemory bool) (Repositories, error) { defer SpinnerStop() syncedRepos := Repositories{} for _, r := range l.PackageRepositories { - repo, err := r.Sync() + repo, err := r.Sync(false) if err != nil { return nil, errors.Wrap(err, "Failed syncing repository: "+r.GetName()) } diff --git a/pkg/installer/interface.go b/pkg/installer/interface.go index 1196a607..1d07c38e 100644 --- a/pkg/installer/interface.go +++ b/pkg/installer/interface.go @@ -48,7 +48,7 @@ type Repository interface { GetTree() tree.Builder SetTree(tree.Builder) Write(path string, resetRevision bool) error - Sync() (Repository, error) + Sync(bool) (Repository, error) GetTreePath() string SetTreePath(string) GetType() string diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 23ee2789..88ad8d5a 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -26,15 +26,16 @@ import ( "strings" "time" - "github.com/mudler/luet/pkg/installer/client" - - "github.com/ghodss/yaml" "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" + "github.com/mudler/luet/pkg/installer/client" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" tree "github.com/mudler/luet/pkg/tree" + + "github.com/ghodss/yaml" + . "github.com/logrusorgru/aurora" "github.com/pkg/errors" ) @@ -74,7 +75,7 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src, } return NewLuetSystemRepository( - config.NewLuetRepository(name, t, descr, urls, priority, true), + config.NewLuetRepository(name, t, descr, urls, priority, true, false), art, tr), nil } @@ -106,6 +107,7 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos p.Urls, p.Priority, true, + false, ), } if p.Revision > 0 { @@ -288,43 +290,88 @@ func (r *LuetSystemRepository) Client() Client { return nil } -func (r *LuetSystemRepository) Sync() (Repository, error) { - Debug("Sync of the repository", r.Name, "in progress..") +func (r *LuetSystemRepository) Sync(force bool) (Repository, error) { + var repoUpdated bool = false + var treefs string + + Debug("Sync of the repository", r.Name, "in progress...") c := r.Client() if c == nil { return nil, errors.New("No client could be generated from repository.") } + + // Retrieve remote repository.yaml for retrieve revision and date file, err := c.DownloadFile(REPOSITORY_SPECFILE) if err != nil { return nil, errors.Wrap(err, "While downloading "+REPOSITORY_SPECFILE) } - repo, err := r.ReadSpecFile(file, true) + + repobasedir := helpers.GetRepoDatabaseDirPath(r.GetName()) + repo, err := r.ReadSpecFile(file, false) if err != nil { return nil, err } + // Remove temporary file that contains repository.html. + // Example: /tmp/HttpClient236052003 + defer os.RemoveAll(file) - archivetree, err := c.DownloadFile(TREE_TARBALL) - if err != nil { - return nil, errors.Wrap(err, "While downloading "+TREE_TARBALL) + if r.Cached { + if !force { + localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, REPOSITORY_SPECFILE), false) + if localRepo != nil { + if localRepo.GetRevision() == repo.GetRevision() && + localRepo.GetLastUpdate() == repo.GetLastUpdate() { + repoUpdated = true + } + } + } + if r.GetTreePath() == "" { + treefs = filepath.Join(repobasedir, "treefs") + } else { + treefs = r.GetTreePath() + } + + } else { + treefs, err = ioutil.TempDir(os.TempDir(), "treefs") + if err != nil { + return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") + } } - defer os.RemoveAll(archivetree) // clean up - treefs, err := ioutil.TempDir(os.TempDir(), "treefs") - if err != nil { - return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") - } - //defer os.RemoveAll(treefs) // clean up + if !repoUpdated { + archivetree, err := c.DownloadFile(TREE_TARBALL) + if err != nil { + return nil, errors.Wrap(err, "While downloading "+TREE_TARBALL) + } + defer os.RemoveAll(archivetree) // clean up - // TODO: Following as option if archive as output? - // archive, err := ioutil.TempDir(os.TempDir(), "archive") - // if err != nil { - // return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs") - // } - // defer os.RemoveAll(archive) // clean up + Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.") - err = helpers.Untar(archivetree, treefs, false) - if err != nil { - return nil, errors.Wrap(err, "Error met while unpacking rootfs") + if r.Cached { + // Copy updated repository.yaml file to repo dir now that the tree is synced. + err = helpers.CopyFile(file, filepath.Join(repobasedir, REPOSITORY_SPECFILE)) + if err != nil { + return nil, errors.Wrap(err, "Error on update "+REPOSITORY_SPECFILE) + } + // Remove previous tree + os.RemoveAll(treefs) + } + Debug("Untar tree of the repository " + r.Name + "...") + err = helpers.Untar(archivetree, treefs, true) + if err != nil { + return nil, errors.Wrap(err, "Error met while unpacking rootfs") + } + + tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64) + + InfoC( + Bold(Red(":house: Repository "+r.GetName()+" revision: ")).String() + + Bold(Green(repo.GetRevision())).String() + " - " + + Bold(Green(time.Unix(tsec, 0).String())).String(), + ) + + } else { + Info("Repository", r.GetName(), "is already up to date.") } reciper := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false)) @@ -332,6 +379,7 @@ func (r *LuetSystemRepository) Sync() (Repository, error) { if err != nil { return nil, errors.Wrap(err, "Error met while unpacking rootfs") } + repo.SetTree(reciper) repo.SetTreePath(treefs) repo.SetUrls(r.GetUrls())