Tag 0.19.2

Fixes https://github.com/mudler/luet/issues/250 as now it is documented
in the cli --help too.

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,49 @@
+{{ if .Versions -}}
+{{ if .Unreleased.CommitGroups -}}
+{{ range .Unreleased.CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Unreleased.Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+
+{{ if .CommitGroups -}}
+{{ range .CommitGroups -}}
+### {{ .Title }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+{{ else }}
+{{ range .Commits -}}
+- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} (https://github.com/mudler/luet/commit/{{.Hash.Short}})
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
+
+{{- if .Versions }}
+[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD
+{{ range .Versions -}}
+{{ if .Tag.Previous -}}
+[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/mudler/luet
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+     title_maps:
+       feat: Features
+       fix: Bug Fixes
+       perf: Performance Improvements
+       refactor: Code Refactoring
+       ci: Continous Integration
+  header:
+    pattern: "(.*)"
+    pattern_maps:
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all bug reports. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Luet version:**
+<!-- Provide the output from "luet --version" -->
+
+**CPU architecture, OS, and Version:**
+<!-- Provide the output from "uname -a"  -->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+<!-- Steps to reproduce the behavior, including the luet command used -->
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Logs**
+<!-- If applicable, add logs with the "--debug" flag enabled to help explain your problem. -->
+
+**Additional context**
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: mudler
+
+---
+
+<!-- Thanks for helping us to improve Luet! We welcome all feature requests. Please fill out each area of the template so we can better help you. Comments like this will be hidden when you post but you can delete them if you wish. -->
+
+**Is your feature request related to a problem? Please describe.**
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]  -->
+
+**Describe the solution you'd like**
+<!-- A clear and concise description of what you want to happen.  -->
+
+**Describe alternatives you've considered**
+<!-- A clear and concise description of any alternative solutions or features you've considered.  -->
+
+**Additional context**
+<!-- Add any other context or screenshots about the feature request here. -->

.github/workflows/image.yml

@@ -0,0 +1,67 @@
+---
+name: 'build container images'
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=quay.io/luet/base
+          VERSION=latest
+          SHORTREF=${GITHUB_SHA::8}
+
+          # If this is git tag, use the tag name as a docker tag
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:${SHORTREF}"
+
+          # If the VERSION looks like a version number, assume that
+          # this is the most recent version of the image and also
+          # tag it 'latest'.
+          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:latest"
+          fi
+
+          # Set output parameters.
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: quay.io
+          username: ${{ secrets.DOCKER_RELEASE_USERNAME }}
+          password: ${{ secrets.DOCKER_RELEASE_PASSWORD }}
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

.github/workflows/pr.yml

@@ -0,0 +1,58 @@
+
+on: pull_request
+name: Build and Test
+jobs:
+  tests-integration:
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+    - name: Tests with Img backend
+      run: sudo -E env "PATH=$PATH" env "LUET_BACKEND=img" make test-integration
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-integration
+  tests-unit:
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: setup-docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+    - name: Install deps
+      run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+            make build
+            sudo cp -rf luet /usr/bin/luet
+    - name: Install GoReleaser
+      uses: goreleaser/goreleaser-action@v2
+      with:
+        install-only: true
+    - name: Build
+      run: sudo -E env "PATH=$PATH" make multiarch-build-small
+    - name: Tests
+      run: sudo -E env "PATH=$PATH" make test-coverage

.github/workflows/push.yml

@@ -0,0 +1,76 @@
+on: push
+concurrency:
+  group: registries-tests
+
+name: Build on push
+jobs:
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+            sudo -E env "PATH=$PATH" \
+                    env "LUET_BACKEND=img" \
+                    make test-integration
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-integration
+
+  tests-unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+            sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+            sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+            sudo chmod a+x "/usr/bin/img"
+            make build
+            sudo cp -rf luet /usr/bin/luet
+      - name: Install GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          install-only: true
+      - name: Build test
+        run: sudo -E env "PATH=$PATH" make multiarch-build-small
+      - name: Tests
+        run: |
+              sudo -E \
+                    env "PATH=$PATH" \
+                    env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                    env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                    make test-coverage

.github/workflows/release.yml

@@ -0,0 +1,94 @@
+on:
+  push:
+    tags:
+      - '*'  # only test and release when a tag is pushed
+concurrency:
+  group: registries-tests
+
+name: Test and Release on tag
+jobs:
+  tests-integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+          sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+          sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+          sudo chmod a+x "/usr/bin/img"
+      - name: Login to quay with img
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo img login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Tests with Img backend
+        run: |
+          sudo -E env "PATH=$PATH" \
+                  env "LUET_BACKEND=img" \
+                  make test-integration
+      - name: Tests
+        run: |
+          sudo -E \
+                env "PATH=$PATH" \
+                env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                make test-integration
+
+  tests-unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: setup-docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+      - name: Login to quay
+        run: echo ${{ secrets.DOCKER_TESTING_PASSWORD }} | sudo -E docker login -u ${{ secrets.DOCKER_TESTING_USERNAME }} --password-stdin quay.io
+      - name: Install deps
+        run: |
+          sudo apt-get install -y upx && sudo -E env "PATH=$PATH" make deps
+          sudo curl -fSL "https://github.com/genuinetools/img/releases/download/v0.5.11/img-linux-amd64" -o "/usr/bin/img"
+          sudo chmod a+x "/usr/bin/img"
+          make build
+          sudo cp -rf luet /usr/bin/luet
+      - name: Tests
+        run: |
+          sudo -E \
+                env "PATH=$PATH" \
+                env "TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE=${{ secrets.DOCKER_TESTING_IMAGE }}" \
+                env "UNIT_TEST_DOCKER_IMAGE_REPOSITORY=${{ secrets.DOCKER_TESTING_UNIT_TEST_IMAGE }}" \
+                make test-coverage
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: [ "tests-integration","tests-unit" ]
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,2 +1,5 @@
 *.swp
 luet
+tests/integration/shunit2
+tests/integration/bin
+release/

.goreleaser.yml

@@ -0,0 +1,43 @@
+before:
+  hooks:
+    - go mod tidy
+dist: release
+source:
+  enabled: true
+  name_template: '{{ .ProjectName }}-{{ .Tag }}-source'
+checksum:
+  name_template: '{{ .ProjectName }}-{{ .Tag }}-checksums.txt'
+builds:
+  -
+    env:
+      - CGO_ENABLED=0
+    ldflags:
+      - -s -w
+      - -X "github.com/mudler/luet/cmd.BuildTime={{ time "2006-01-02 15:04:05 MST" }}"
+      - -X "github.com/mudler/luet/cmd.BuildCommit={{ .FullCommit }}"
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm
+      - arm64
+      - 386
+    goarm:
+      - 6
+      - 7
+archives:
+  - format: binary # this removes the tar of the archives, leaving the binaries alone
+    name_template: luet-{{ .Tag }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+      - '^Merge pull request'
+release:
+  header: |
+    Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages.
+
+    It has zero dependencies and it is well suitable for "from scratch" environments.
+    It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded device

.travis.yml

@@ -1,18 +1,35 @@
+dist: bionic
 language: go
-services:
-  - docker
 go:
-  - "1.12"
+  - "1.14"
+
 env:
-  - "GO15VENDOREXPERIMENT=1"
+  global:
+    - "GO15VENDOREXPERIMENT=1"
+  jobs:
+    - "DOCKER_BUILDKIT=0"
+    - "DOCKER_BUILDKIT=1"
 before_install:
-  - make deps
-  - curl -LO https://storage.googleapis.com/container-diff/latest/container-diff-linux-amd64 && chmod +x container-diff-linux-amd64 && mkdir -p $HOME/bin && export PATH=$PATH:$HOME/bin && mv container-diff-linux-amd64 $HOME/bin/container-diff
+  - sudo rm -rf /var/lib/apt/lists/*
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge"
+  - sudo apt-get update
+  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+  - export DOCKER_CLI_EXPERIMENTAL=enabled
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - mkdir -vp ~/.docker/cli-plugins/
+  - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+  - chmod a+x ~/.docker/cli-plugins/docker-buildx
+  - docker buildx version
+  - sudo -E env "PATH=$PATH" apt-get install -y libcap2-bin
+  - sudo -E env "PATH=$PATH" make deps
 script:
-  - make multiarch-build test
-after_success:
-  - make coverage
-  - bash <(curl -s https://codecov.io/bash)
-  - git config --global user.name "Deployer" && git config --global user.email foo@bar.com
-  - go get github.com/tcnksm/ghr
-  - ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+  - sudo -E env "PATH=$PATH" make multiarch-build test-integration test-coverage
+
+#after_success:
+#  - |
+#      if [ -n "$TRAVIS_TAG" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+#        sudo -E env "PATH=$PATH" git config --global user.name "Deployer" && git config --global user.email foo@bar.com
+#        sudo -E env "PATH=$PATH" go get github.com/tcnksm/ghr
+#        sudo -E env "PATH=$PATH" ghr -u mudler -r luet --replace $TRAVIS_TAG release/
+#      fi

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+
+We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's:
+
+- Reporting a bug
+- Discussing the current state of the code
+- Submitting a fix
+- Proposing new features
+- Becoming a maintainer
+
+## We Develop with Github
+We use github to host code, to track issues and feature requests, as well as accept pull requests.
+
+## Stay in touch
+
+Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDUxLWQ5ODVlNTI1MTYzNDRkYzkyYmM1YWE5YjM0NTliNDEzNmQwMTkxNDRhNDIzM2Y5NDBlOTZjZTYxYWQyNDE4YzY#/) and hang out with the community! It will be much easier to get started and do your first steps in contributing to the project.
+
+## All Code Changes Happen Through Pull Requests
+Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
+
+1. Fork the repo you want to contribute to and create your branch from `develop`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. Issue that pull request!
+
+## Any contributions you make will be under the Software License of the repository
+In short, when you submit code changes, your submissions are understood to be under the same License that covers the project. Feel free to contact the maintainers if that's a concern.
+
+## Report bugs using Github's [issues](https://github.com/mudler/luet/issues)
+We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/mudler/luet/issues/new); it's that easy!
+
+## Write bug reports with detail, background, and sample code
+Try to be as more descriptive as possible. When opening a new issue you will be prompted to choose between a bug or a feature request, with a small template to fill details with. Be specific!
+
+**Great Bug Reports** tend to have:
+
+- A quick summary and/or background
+- Steps to reproduce
+  - Be specific!
+  - Give sample code if you can. 
+- What you expected would happen
+- What actually happens
+- Notes (possibly including why you think this might be happening, or stuff you tried that didn't work)
+
+People *love* thorough bug reports.
+
+
+## License
+By contributing, you agree that your contributions will be licensed under the project Licenses.
+
+## References
+This document was adapted from the open-source contribution guidelines from https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62

Dockerfile

@@ -0,0 +1,11 @@
+FROM golang as builder
+RUN apt-get update && apt-get install -y upx
+ADD . /luet
+RUN cd /luet && make build-small
+
+FROM scratch
+ENV LUET_NOLOCK=true
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /luet/luet /usr/bin/luet
+
+ENTRYPOINT ["/usr/bin/luet"]

Makefile

@@ -1,11 +1,11 @@
+
+# go tool nm ./luet | grep Commit
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S %Z')"
+override LDFLAGS += -X "github.com/mudler/luet/cmd.BuildCommit=$(shell git rev-parse HEAD)"
+
 NAME ?= luet
 PACKAGE_NAME ?= $(NAME)
 PACKAGE_CONFLICT ?= $(PACKAGE_NAME)-beta
-REVISION := $(shell git rev-parse --short HEAD || echo dev)
-VERSION := $(shell git describe --tags || echo $(REVISION))
-VERSION := $(shell echo $(VERSION) | sed -e 's/^v//g')
-ITTERATION := $(shell date +%s)
-BUILD_PLATFORMS ?= -osarch="linux/amd64" -osarch="linux/386" -osarch="linux/arm"
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 .PHONY: all
@@ -19,12 +19,20 @@ fmt:
 test:
 	GO111MODULE=off go get github.com/onsi/ginkgo/ginkgo
 	GO111MODULE=off go get github.com/onsi/gomega/...
-	ginkgo -race -r ./...
+	ginkgo -race -r -flakeAttempts 3 ./...
+
+.PHONY: test-integration
+test-integration:
+	tests/integration/run.sh
 
 .PHONY: coverage
 coverage:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
 
+.PHONY: test-coverage
+test-coverage:
+	scripts/ginkgo.coverage.sh --codecov
+
 .PHONY: help
 help:
 	# make all => deps test lint build
@@ -36,6 +44,8 @@ help:
 .PHONY: clean
 clean:
 	rm -rf release/
+	rm -rf tests/integration/shunit2
+	rm -rf tests/integration/bin
 
 .PHONY: deps
 deps:
@@ -49,16 +59,19 @@ deps:
 
 .PHONY: build
 build:
-	go build
+	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)'
 
-.PHONY: gox-build
-gox-build:
-	# Building gitlab-ci-multi-runner for $(BUILD_PLATFORMS)
-	gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+.PHONY: build-small
+build-small:
+	@$(MAKE) LDFLAGS+="-s -w" build
+	upx --brute -1 $(NAME)
+
+.PHONY: image
+image:
+	docker build --rm -t luet/base .
 
 .PHONY: lint
 lint:
-	# Checking project code style...
 	golint ./... | grep -v "be unexported"
 
 .PHONY: vendor
@@ -71,6 +84,11 @@ test-docker:
 				--workdir /go/src/github.com/mudler/luet -ti golang:latest \
 				bash -c "make test"
 
-.PHONY: multiarch-build
 multiarch-build:
-	gox $(BUILD_PLATFORMS) -output="release/$(NAME)-$(VERSION)-{{.OS}}-{{.Arch}}"
+	goreleaser build --snapshot --rm-dist
+
+multiarch-build-small:
+	@$(MAKE) multiarch-build
+	for file in $(ROOT_DIR)/release/**/* ; do \
+		upx --brute -1 $${file} ; \
+	done

README.md

@@ -1,9 +1,84 @@
-# luet - Package manager
-[![Go Report Card](https://goreportcard.com/badge/github.com/mudler/luet)](https://goreportcard.com/report/github.com/mudler/luet)
-[![Build Status](https://travis-ci.org/mudler/luet.svg?branch=master)](https://travis-ci.org/mudler/luet)
+
+<p align="center">
+  <img width=150 height=150 src="https://user-images.githubusercontent.com/2420543/119691600-0293d700-be4b-11eb-827f-49ff1174a07a.png">
+</p>
+
+# luet - Container-based Package manager
+
+[![Docker Repository on Quay](https://quay.io/repository/luet/base/status "Docker Repository on Quay")](https://quay.io/repository/luet/base)
+[![Build and release on push](https://github.com/mudler/luet/actions/workflows/release.yml/badge.svg)](https://github.com/mudler/luet/actions/workflows/release.yml)
 [![GoDoc](https://godoc.org/github.com/mudler/luet?status.svg)](https://godoc.org/github.com/mudler/luet)
 [![codecov](https://codecov.io/gh/mudler/luet/branch/master/graph/badge.svg)](https://codecov.io/gh/mudler/luet)
 
-No dep hell for your own package manager !
+Luet is a multi-platform Package Manager based off from containers - it uses Docker (and others) to build packages. It has zero dependencies and it is well suitable for "from scratch" environments. It can also version entire rootfs and enables delivery of OTA-alike updates, making it a perfect fit for the Edge computing era and IoT embedded devices.
 
-Package manager that reuses Gentoo's Portage output binaries, but which doesn't depend on it.
+It offers a simple [specfile format](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/) in YAML notation to define both [packages](https://luet-lab.github.io/docs/docs/concepts/packages/) and [rootfs](https://luet-lab.github.io/docs/docs/concepts/packages/#package-layers). As it is based on containers, it can be also used to build stages for Linux From Scratch installations and it can build and track updates for those systems.
+
+It is written entirely in Golang and where used as package manager, it can run in from scratch environment, with zero dependencies.
+
+[![asciicast](https://asciinema.org/a/388348.svg)](https://asciinema.org/a/388348)
+
+
+## In a glance
+
+- Luet can reuse Gentoo's portage tree hierarchy, and it is heavily inspired from it.
+- It builds, installs, uninstalls and perform upgrades on machines
+- Installer doesn't depend on anything ( 0 dep installer !), statically built
+- You can install it aside also with your current distro package manager, and start building and distributing your packages
+- [Support for packages as "layers"](https://luet-lab.github.io/docs/docs/concepts/packages/specfile/#building-strategies)
+- [It uses SAT solving techniques to solve the deptree](https://luet-lab.github.io/docs/docs/concepts/overview/constraints/) ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
+- Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
+- [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
+- [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
+
+## Install
+
+To install luet, you can grab a release on the [Release page](https://github.com/mudler/luet/releases) or to install it in your system:
+
+```bash
+$ curl https://get.mocaccino.org/luet/get_luet_root.sh | sudo sh
+$ luet search ...
+$ luet install ..
+$ luet --help
+```
+
+## Build from source
+
+```bash
+$ git clone https://github.com/mudler/luet.git
+$ cd luet
+$ make build
+```
+
+## Documentation
+
+[Documentation](https://luet-lab.github.io/docs) is available, or
+run `luet --help`,  any subcommand is documented as well, try e.g.: `luet build --help`.
+
+# Dependency solving
+
+Luet uses SAT and Reinforcement learning engine for dependency solving.
+It encodes the package requirements into a SAT problem, using [gophersat](https://github.com/crillab/gophersat) to solve the dependency tree and give a concrete model as result.
+
+## SAT encoding
+
+Each package and its constraints are encoded and built around [OPIUM](https://ranjitjhala.github.io/static/opium.pdf). Additionally, Luet treats
+also selectors seamlessly while building the model, adding *ALO* ( *At least one* ) and *AMO* ( *At most one* ) rules to guarantee coherence within the installed system.
+
+## Reinforcement learning
+
+Luet also implements a small and portable qlearning agent that will try to solve conflict on your behalf
+when they arises while trying to validate your queries against the system model.
+
+To leverage it, simply pass ```--solver-type qlearning``` to the subcommands that supports it ( you can check out by invoking ```--help``` ).
+
+
+## Authors
+
+Luet is here thanks to our amazing [contributors](https://github.com/mudler/luet/graphs/contributors)!.
+
+Luet was originally created by Ettore Di Giacinto, mudler@sabayon.org, mudler@gentoo.org.
+
+## License
+
+Luet is distributed under the terms of GPLv3, check out the LICENSE file.

cmd/box.go

@@ -0,0 +1,36 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/box"
+
+	"github.com/spf13/cobra"
+)
+
+var boxGroupCmd = &cobra.Command{
+	Use:   "box [command] [OPTIONS]",
+	Short: "Manage luet boxes",
+}
+
+func init() {
+	RootCmd.AddCommand(boxGroupCmd)
+
+	boxGroupCmd.AddCommand(
+		NewBoxExecCommand(),
+	)
+}

cmd/box/exec.go

@@ -0,0 +1,81 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_box
+
+import (
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/box"
+
+	"github.com/spf13/cobra"
+)
+
+func NewBoxExecCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "exec [OPTIONS]",
+		Short: "Execute a binary in a box",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			stdin, _ := cmd.Flags().GetBool("stdin")
+			stdout, _ := cmd.Flags().GetBool("stdout")
+			stderr, _ := cmd.Flags().GetBool("stderr")
+			rootfs, _ := cmd.Flags().GetString("rootfs")
+			base, _ := cmd.Flags().GetBool("decode")
+			entrypoint, _ := cmd.Flags().GetString("entrypoint")
+			envs, _ := cmd.Flags().GetStringArray("env")
+			mounts, _ := cmd.Flags().GetStringArray("mount")
+
+			if base {
+				var ss []string
+				for _, a := range args {
+					sDec, _ := b64.StdEncoding.DecodeString(a)
+					ss = append(ss, string(sDec))
+				}
+				//If the command to run is complex,using base64 to avoid bad input
+
+				args = ss
+			}
+			util.DefaultContext.Info("Executing", args, "in", rootfs)
+
+			b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+			err := b.Run()
+			if err != nil {
+				util.DefaultContext.Fatal(err)
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	ans.Flags().String("rootfs", path, "Rootfs path")
+	ans.Flags().Bool("stdin", false, "Attach to stdin")
+	ans.Flags().Bool("stdout", true, "Attach to stdout")
+	ans.Flags().Bool("stderr", true, "Attach to stderr")
+	ans.Flags().Bool("decode", false, "Base64 decode")
+	ans.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	ans.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	ans.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	return ans
+}

cmd/build.go

@@ -15,15 +15,22 @@
 package cmd
 
 import (
-	"io/ioutil"
+	"fmt"
 	"os"
-	"regexp"
-	"runtime"
+	"path/filepath"
 
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
 	"github.com/mudler/luet/pkg/compiler"
-	"github.com/mudler/luet/pkg/compiler/backend"
-	. "github.com/mudler/luet/pkg/logger"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/installer"
+
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
 	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
 	tree "github.com/mudler/luet/pkg/tree"
 
 	"github.com/spf13/cobra"
@@ -33,104 +40,247 @@ import (
 var buildCmd = &cobra.Command{
 	Use:   "build <package name> <package name> <package name> ...",
 	Short: "build a package or a tree",
-	Long:  `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`,
+	Long: `Builds one or more packages from a tree (current directory is implied):
+
+	$ luet build utils/busybox utils/yq ...
+
+Builds all packages
+
+	$ luet build --all
+
+Builds only the leaf packages:
+
+	$ luet build --full
+
+Build package revdeps:
+
+	$ luet build --revdeps utils/yq
+
+Build package without dependencies (needs the images already in the host, or either need to be available online):
+
+	$ luet build --nodeps utils/yq ...
+
+Build packages specifying multiple definition trees:
+
+	$ luet build --tree overlay/path --tree overlay/path2 utils/yq ...
+`, PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
+		viper.BindPFlag("privileged", cmd.Flags().Lookup("privileged"))
+		viper.BindPFlag("revdeps", cmd.Flags().Lookup("revdeps"))
+		viper.BindPFlag("all", cmd.Flags().Lookup("all"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		util.BindValuesFlags(cmd)
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+
+		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
+		viper.BindPFlag("pull", cmd.Flags().Lookup("pull"))
+		viper.BindPFlag("wait", cmd.Flags().Lookup("wait"))
+		viper.BindPFlag("keep-images", cmd.Flags().Lookup("keep-images"))
+
+		util.BindSolverFlags(cmd)
+
+		viper.BindPFlag("general.show_build_output", cmd.Flags().Lookup("live-output"))
+		viper.BindPFlag("backend-args", cmd.Flags().Lookup("backend-args"))
+
+	},
 	Run: func(cmd *cobra.Command, args []string) {
 
-		src := viper.GetString("tree")
-		dst := viper.GetString("output")
-		concurrency := viper.GetInt("concurrency")
+		treePaths := viper.GetStringSlice("tree")
+		dst := viper.GetString("destination")
+		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
 		backendType := viper.GetString("backend")
 		privileged := viper.GetBool("privileged")
 		revdeps := viper.GetBool("revdeps")
 		all := viper.GetBool("all")
-		databaseType := viper.GetString("database")
+		compressionType := viper.GetString("compression")
+		imageRepository := viper.GetString("image-repository")
+		values := util.ValuesFlags()
+		wait := viper.GetBool("wait")
+		push := viper.GetBool("push")
+		pull := viper.GetBool("pull")
+		keepImages := viper.GetBool("keep-images")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		onlyTarget, _ := cmd.Flags().GetBool("only-target-package")
+		full, _ := cmd.Flags().GetBool("full")
+		rebuild, _ := cmd.Flags().GetBool("rebuild")
 
-		compilerSpecs := compiler.NewLuetCompilationspecs()
-		var compilerBackend compiler.CompilerBackend
+		var results Results
+		backendArgs := viper.GetStringSlice("backend-args")
+
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
+		}
+		pretend, _ := cmd.Flags().GetBool("pretend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
+		compilerSpecs := compilerspec.NewLuetCompilationspecs()
 		var db pkg.PackageDatabase
-		switch backendType {
-		case "img":
-			compilerBackend = backend.NewSimpleImgBackend()
-		case "docker":
-			compilerBackend = backend.NewSimpleDockerBackend()
-		}
 
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
+		compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType)
+		helpers.CheckErr(err)
+
+		db = pkg.NewInMemoryDatabase(false)
 		defer db.Clean()
 
 		generalRecipe := tree.NewCompilerRecipe(db)
 
-		Info("Loading", src)
-		err := generalRecipe.Load(src)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+		if fromRepo {
+			if err := installer.LoadBuildTree(generalRecipe, db, util.DefaultContext); err != nil {
+				util.DefaultContext.Warning("errors while loading trees from repositories", err.Error())
+			}
 		}
-		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
 
-		err = luetCompiler.Prepare(concurrency)
-		if err != nil {
-			Fatal("Error: " + err.Error())
+		for _, src := range treePaths {
+			util.DefaultContext.Info("Loading tree", src)
+			helpers.CheckErr(generalRecipe.Load(src))
 		}
-		if !all {
+
+		util.DefaultContext.Info("Building in", dst)
+
+		opts := util.SetSolverConfig(util.DefaultContext)
+		pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
+
+		util.DefaultContext.Config.GetGeneral().ShowBuildOutput = viper.GetBool("general.show_build_output")
+
+		util.DefaultContext.Debug("Solver", opts.CompactString())
+
+		opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency}
+
+		luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(),
+			options.NoDeps(nodeps),
+			options.WithBackendType(backendType),
+			options.PushImages(push),
+			options.WithBuildValues(values),
+			options.WithPullRepositories(pullRepo),
+			options.WithPushRepository(imageRepository),
+			options.Rebuild(rebuild),
+			options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, fromRepo, treePaths)),
+			options.WithSolverOptions(*opts),
+			options.Wait(wait),
+			options.OnlyTarget(onlyTarget),
+			options.PullFirst(pull),
+			options.KeepImg(keepImages),
+			options.OnlyDeps(onlydeps),
+			options.WithContext(util.DefaultContext),
+			options.BackendArgs(backendArgs),
+			options.Concurrency(concurrency),
+			options.WithCompressionType(compression.Implementation(compressionType)),
+		)
+
+		if full {
+			specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			for _, spec := range specs {
+				util.DefaultContext.Info(":package: Selecting ", spec.GetPackage().GetName(), spec.GetPackage().GetVersion())
+
+				compilerSpecs.Add(spec)
+			}
+		} else if !all {
 			for _, a := range args {
-				decodepackage, err := regexp.Compile(`^([<>]?\~?=?)((([^\/]+)\/)?(?U)(\S+))(-(\d+(\.\d+)*[a-z]?(_(alpha|beta|pre|rc|p)\d*)*(-r\d+)?))?$`)
+				pack, err := helpers.ParsePackageStr(a)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
 				}
-				packageInfo := decodepackage.FindAllStringSubmatch(a, -1)
 
-				category := packageInfo[0][4]
-				name := packageInfo[0][5]
-				version := packageInfo[0][7]
-				spec, err := luetCompiler.FromPackage(&pkg.DefaultPackage{Name: name, Category: category, Version: version})
+				spec, err := luetCompiler.FromPackage(pack)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					util.DefaultContext.Fatal("Error: " + err.Error())
 				}
 
 				spec.SetOutputPath(dst)
 				compilerSpecs.Add(spec)
 			}
 		} else {
-			w, e := generalRecipe.Tree().World()
-			if e != nil {
-				Fatal("Error: " + err.Error())
-			}
+			w := generalRecipe.GetDatabase().World()
+
 			for _, p := range w {
 				spec, err := luetCompiler.FromPackage(p)
 				if err != nil {
-					Fatal("Error: " + err.Error())
+					util.DefaultContext.Fatal("Error: " + err.Error())
 				}
-				Info(":package: Selecting ", p.GetName(), p.GetVersion())
+				util.DefaultContext.Info(":package: Selecting ", p.GetName(), p.GetVersion())
+				spec.SetOutputPath(dst)
 				compilerSpecs.Add(spec)
 			}
 		}
 
-		var artifact []compiler.Artifact
+		var artifact []*artifact.PackageArtifact
 		var errs []error
 		if revdeps {
-			artifact, errs = luetCompiler.CompileWithReverseDeps(concurrency, privileged, compilerSpecs)
+			artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs)
 
+		} else if pretend {
+			toCalculate := []*compilerspec.LuetCompilationSpec{}
+			if full {
+				var err error
+				toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...)
+				if err != nil {
+					errs = append(errs, err)
+				}
+			} else {
+				toCalculate = compilerSpecs.All()
+			}
+
+			for _, sp := range toCalculate {
+				ht := compiler.NewHashTree(generalRecipe.GetDatabase())
+				hashTree, err := ht.Query(luetCompiler, sp)
+				if err != nil {
+					errs = append(errs, err)
+				}
+				for _, p := range hashTree.Dependencies {
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       p.Package.GetName(),
+							Version:    p.Package.GetVersion(),
+							Category:   p.Package.GetCategory(),
+							Repository: "",
+							Hidden:     p.Package.IsHidden(),
+							Target:     sp.GetPackage().HumanReadableString(),
+						})
+				}
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			case "terminal":
+				for _, p := range results.Packages {
+					util.DefaultContext.Info(p.String())
+				}
+			}
 		} else {
-			artifact, errs = luetCompiler.CompileParallel(concurrency, privileged, compilerSpecs)
 
+			artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs)
 		}
 		if len(errs) != 0 {
 			for _, e := range errs {
-				Error("Error: " + e.Error())
+				util.DefaultContext.Error("Error: " + e.Error())
 			}
-			Fatal("Bailing out")
+			util.DefaultContext.Fatal("Bailing out")
 		}
 		for _, a := range artifact {
-			Info("Artifact generated:", a.GetPath())
+			util.DefaultContext.Info("Artifact generated:", a.Path)
 		}
 	},
 }
@@ -138,24 +288,40 @@ var buildCmd = &cobra.Command{
 func init() {
 	path, err := os.Getwd()
 	if err != nil {
-		Fatal(err)
+		util.DefaultContext.Fatal(err)
 	}
-	buildCmd.Flags().String("tree", path, "Source luet tree")
-	viper.BindPFlag("tree", buildCmd.Flags().Lookup("tree"))
-	buildCmd.Flags().String("output", path, "Destination folder")
-	viper.BindPFlag("output", buildCmd.Flags().Lookup("output"))
-	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
-	viper.BindPFlag("backend", buildCmd.Flags().Lookup("backend"))
-	buildCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
-	viper.BindPFlag("concurrency", buildCmd.Flags().Lookup("concurrency"))
-	buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)")
-	viper.BindPFlag("privileged", buildCmd.Flags().Lookup("privileged"))
-	buildCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", buildCmd.Flags().Lookup("database"))
-	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
-	viper.BindPFlag("revdeps", buildCmd.Flags().Lookup("revdeps"))
 
-	buildCmd.Flags().Bool("all", false, "Build all packages in the tree")
-	viper.BindPFlag("all", buildCmd.Flags().Lookup("all"))
+	buildCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the tree to use.")
+	buildCmd.Flags().String("backend", "docker", "backend used (docker,img)")
+	buildCmd.Flags().Bool("privileged", true, "Privileged (Keep permissions)")
+	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
+	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
+	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+	buildCmd.Flags().StringSlice("values", []string{}, "Build values file to interpolate with each package")
+	buildCmd.Flags().StringSliceP("backend-args", "a", []string{}, "Backend args")
+
+	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
+	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
+	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	buildCmd.Flags().Bool("push", false, "Push images to a hub")
+	buildCmd.Flags().Bool("pull", false, "Pull images from a hub")
+	buildCmd.Flags().Bool("wait", false, "Don't build all intermediate images, but wait for them until they are available")
+	buildCmd.Flags().Bool("keep-images", true, "Keep built docker images in the host")
+	buildCmd.Flags().Bool("nodeps", false, "Build only the target packages, skipping deps (it works only if you already built the deps locally, or by using --pull) ")
+	buildCmd.Flags().Bool("onlydeps", false, "Build only package dependencies")
+	buildCmd.Flags().Bool("only-target-package", false, "Build packages of only the required target. Otherwise builds all the necessary ones not present in the destination")
+	buildCmd.Flags().String("solver-type", "", "Solver strategy")
+	buildCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	buildCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	buildCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	buildCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	buildCmd.Flags().Bool("live-output", util.DefaultContext.Config.GetGeneral().ShowBuildOutput, "Enable live output of the build phase.")
+	buildCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	buildCmd.Flags().Bool("rebuild", false, "To combine with --pull. Allows to rebuild the target package even if an image is available, against a local values file")
+	buildCmd.Flags().Bool("pretend", false, "Just print what packages will be compiled")
+	buildCmd.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
+
+	buildCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+
 	RootCmd.AddCommand(buildCmd)
 }

cmd/cleanup.go

@@ -0,0 +1,73 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/cmd/util"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	"github.com/spf13/cobra"
+)
+
+var cleanupCmd = &cobra.Command{
+	Use:   "cleanup",
+	Short: "Clean packages cache.",
+	Long:  `remove downloaded packages tarballs and clean cache directory`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var cleaned int = 0
+		util.SetSystemConfig(util.DefaultContext)
+		// Check if cache dir exists
+		if fileHelper.Exists(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()) {
+
+			files, err := ioutil.ReadDir(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath())
+			if err != nil {
+				util.DefaultContext.Fatal("Error on read cachedir ", err.Error())
+			}
+
+			for _, file := range files {
+
+				if util.DefaultContext.Config.GetGeneral().Debug {
+					util.DefaultContext.Info("Removing ", file.Name())
+				}
+
+				err := os.RemoveAll(
+					filepath.Join(util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath(), file.Name()))
+				if err != nil {
+					util.DefaultContext.Fatal("Error on removing", file.Name())
+				}
+				cleaned++
+			}
+		}
+
+		util.DefaultContext.Info(fmt.Sprintf("Cleaned: %d files from %s", cleaned, util.DefaultContext.Config.GetSystem().GetSystemPkgsCacheDirPath()))
+
+	},
+}
+
+func init() {
+	cleanupCmd.Flags().String("system-dbpath", "", "System db path")
+	cleanupCmd.Flags().String("system-target", "", "System rootpath")
+	cleanupCmd.Flags().String("system-engine", "", "System DB engine")
+	RootCmd.AddCommand(cleanupCmd)
+}

cmd/config.go

@@ -0,0 +1,43 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+)
+
+var configCmd = &cobra.Command{
+	Use:     "config",
+	Short:   "Print config",
+	Long:    `Show luet configuration`,
+	Aliases: []string{"c"},
+	Run: func(cmd *cobra.Command, args []string) {
+		data, err := util.DefaultContext.Config.YAML()
+		if err != nil {
+			util.DefaultContext.Fatal(err.Error())
+		}
+
+		fmt.Println(string(data))
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(configCmd)
+}

cmd/convert.go

@@ -1,97 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-package cmd
-
-import (
-	"io/ioutil"
-	"runtime"
-
-	. "github.com/mudler/luet/pkg/logger"
-	pkg "github.com/mudler/luet/pkg/package"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	"github.com/mudler/luet/pkg/tree/builder/gentoo"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var convertCmd = &cobra.Command{
-	Use:   "convert",
-	Short: "convert other package manager tree into luet",
-	Long:  `Parses external PM and produces a luet parsable tree`,
-	Run: func(cmd *cobra.Command, args []string) {
-
-		t := viper.GetString("type")
-		c := viper.GetInt("concurrency")
-		databaseType := viper.GetString("database")
-		var db pkg.PackageDatabase
-
-		if len(args) != 2 {
-			Fatal("Incorrect number of arguments")
-		}
-
-		input := args[0]
-		output := args[1]
-		Info("Converting trees from " + input + " [" + t + "]")
-
-		var builder tree.Parser
-		switch t {
-		case "gentoo":
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
-		default: // dup
-			builder = gentoo.NewGentooBuilder(&gentoo.SimpleEbuildParser{}, c, gentoo.InMemory)
-		}
-
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
-		defer db.Clean()
-
-		packageTree, err := builder.Generate(input)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
-		defer packageTree.GetPackageSet().Clean()
-		Info("Tree generated")
-
-		generalRecipe := tree.NewGeneralRecipe(db)
-		Info("Saving generated tree to " + output)
-
-		generalRecipe.WithTree(packageTree)
-		err = generalRecipe.Save(output)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-	},
-}
-
-func init() {
-	convertCmd.Flags().String("type", "gentoo", "source type")
-	viper.BindPFlag("type", convertCmd.Flags().Lookup("type"))
-	convertCmd.Flags().Int("concurrency", runtime.NumCPU(), "Concurrency")
-	viper.BindPFlag("concurrency", convertCmd.Flags().Lookup("concurrency"))
-	convertCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", convertCmd.Flags().Lookup("database"))
-
-	RootCmd.AddCommand(convertCmd)
-}

cmd/create-repo.go

@@ -0,0 +1,202 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	//	. "github.com/mudler/luet/pkg/logger"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var createrepoCmd = &cobra.Command{
+	Use:   "create-repo",
+	Short: "Create a luet repository from a build",
+	Long: `Builds tree metadata from a set of packages and a tree definition:
+
+	$ luet create-repo
+
+Provide specific paths for packages, tree, and metadata output which is generated:
+
+	$ luet create-repo --packages my/packages/path --tree my/tree/path --output my/packages/path ...
+
+Provide name and description of the repository:
+
+	$ luet create-repo --name "foo" --description "bar" ...
+
+Change compression method:
+	
+	$ luet create-repo --tree-compression gzip --meta-compression gzip
+
+Create a repository from the metadata description defined in the luet.yaml config file:
+
+	$ luet create-repo --repo repository1
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
+		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
+		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
+		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
+		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
+		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
+		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
+		viper.BindPFlag("tree-filename", cmd.Flags().Lookup("tree-filename"))
+		viper.BindPFlag("meta-compression", cmd.Flags().Lookup("meta-compression"))
+		viper.BindPFlag("meta-filename", cmd.Flags().Lookup("meta-filename"))
+		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
+		viper.BindPFlag("repo", cmd.Flags().Lookup("repo"))
+		viper.BindPFlag("from-metadata", cmd.Flags().Lookup("from-metadata"))
+		viper.BindPFlag("force-push", cmd.Flags().Lookup("force-push"))
+		viper.BindPFlag("push-images", cmd.Flags().Lookup("push-images"))
+
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var repo *installer.LuetSystemRepository
+
+		treePaths := viper.GetStringSlice("tree")
+		dst := viper.GetString("output")
+
+		name := viper.GetString("name")
+		descr := viper.GetString("descr")
+		urls := viper.GetStringSlice("urls")
+		t := viper.GetString("type")
+		reset := viper.GetBool("reset-revision")
+		treetype := viper.GetString("tree-compression")
+		treeName := viper.GetString("tree-filename")
+		metatype := viper.GetString("meta-compression")
+		metaName := viper.GetString("meta-filename")
+		source_repo := viper.GetString("repo")
+		backendType := viper.GetString("backend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
+
+		treeFile := installer.NewDefaultTreeRepositoryFile()
+		metaFile := installer.NewDefaultMetaRepositoryFile()
+		compilerBackend, err := compiler.NewBackend(util.DefaultContext, backendType)
+		helpers.CheckErr(err)
+		force := viper.GetBool("force-push")
+		imagePush := viper.GetBool("push-images")
+
+		opts := []installer.RepositoryOption{
+			installer.WithSource(viper.GetString("packages")),
+			installer.WithPushImages(imagePush),
+			installer.WithForce(force),
+			installer.FromRepository(fromRepo),
+			installer.WithImagePrefix(dst),
+			installer.WithDatabase(pkg.NewInMemoryDatabase(false)),
+			installer.WithCompilerBackend(compilerBackend),
+			installer.FromMetadata(viper.GetBool("from-metadata")),
+			installer.WithContext(util.DefaultContext),
+		}
+
+		if source_repo != "" {
+			// Search for system repository
+			lrepo, err := util.DefaultContext.Config.GetSystemRepository(source_repo)
+			helpers.CheckErr(err)
+
+			if len(treePaths) <= 0 {
+				treePaths = []string{lrepo.TreePath}
+			}
+
+			if t == "" {
+				t = lrepo.Type
+			}
+
+			opts = append(opts,
+				installer.WithName(lrepo.Name),
+				installer.WithDescription(lrepo.Description),
+				installer.WithType(t),
+				installer.WithUrls(lrepo.Urls...),
+				installer.WithPriority(lrepo.Priority),
+				installer.WithTree(treePaths...),
+			)
+
+		} else {
+			opts = append(opts,
+				installer.WithName(name),
+				installer.WithDescription(descr),
+				installer.WithType(t),
+				installer.WithUrls(urls...),
+				installer.WithTree(treePaths...),
+			)
+		}
+
+		repo, err = installer.GenerateRepository(opts...)
+		helpers.CheckErr(err)
+
+		if treetype != "" {
+			treeFile.SetCompressionType(compression.Implementation(treetype))
+		}
+
+		if treeName != "" {
+			treeFile.SetFileName(treeName)
+		}
+
+		if metatype != "" {
+			metaFile.SetCompressionType(compression.Implementation(metatype))
+		}
+
+		if metaName != "" {
+			metaFile.SetFileName(metaName)
+		}
+
+		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
+		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
+
+		err = repo.Write(util.DefaultContext, dst, reset, true)
+		helpers.CheckErr(err)
+
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	helpers.CheckErr(err)
+
+	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
+	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
+	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
+	createrepoCmd.Flags().String("name", "luet", "Repository name")
+	createrepoCmd.Flags().String("descr", "luet", "Repository description")
+	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
+	createrepoCmd.Flags().String("type", "disk", "Repository type (disk, http, docker)")
+	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
+	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+	createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)")
+
+	createrepoCmd.Flags().Bool("force-push", false, "Force overwrite of docker images if already present online")
+	createrepoCmd.Flags().Bool("push-images", false, "Enable/Disable docker image push for docker repositories")
+	createrepoCmd.Flags().Bool("from-metadata", false, "Consider metadata files from the packages folder while indexing the new tree")
+
+	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
+	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
+	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
+	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+
+	RootCmd.AddCommand(createrepoCmd)
+}

cmd/database.go

@@ -0,0 +1,42 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/database"
+
+	"github.com/spf13/cobra"
+)
+
+var databaseGroupCmd = &cobra.Command{
+	Use:   "database [command] [OPTIONS]",
+	Short: "Manage system database (dangerous commands ahead!)",
+	Long: `Allows to manipulate Luet internal database of installed packages. Use with caution!
+
+Removing packages by hand from the database can result in a broken system, and thus it's not reccomended.
+`,
+}
+
+func init() {
+	RootCmd.AddCommand(databaseGroupCmd)
+
+	databaseGroupCmd.AddCommand(
+		NewDatabaseCreateCommand(),
+		NewDatabaseGetCommand(),
+		NewDatabaseRemoveCommand(),
+	)
+}

cmd/database/create.go

@@ -0,0 +1,89 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"io/ioutil"
+
+	"github.com/mudler/luet/cmd/util"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
+
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseCreateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "create <artifact_metadata1.yaml> <artifact_metadata1.yaml>",
+		Short: "Insert a package in the system DB",
+		Long: `Inserts a package in the system database:
+
+		$ luet database create foo.yaml
+
+"luet database create" injects a package in the system database without actually installing it, use it with caution.
+
+This commands takes multiple yaml input file representing package artifacts, that are usually generated while building packages.
+
+The yaml must contain the package definition, and the file list at least.
+
+For reference, inspect a "metadata.yaml" file generated while running "luet build"`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			util.BindSystemFlags(cmd)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			util.SetSystemConfig(util.DefaultContext)
+			systemDB := util.DefaultContext.Config.GetSystemDB()
+
+			for _, a := range args {
+				dat, err := ioutil.ReadFile(a)
+				if err != nil {
+					util.DefaultContext.Fatal("Failed reading ", a, ": ", err.Error())
+				}
+				art, err := artifact.NewPackageArtifactFromYaml(dat)
+				if err != nil {
+					util.DefaultContext.Fatal("Failed reading yaml ", a, ": ", err.Error())
+				}
+
+				files := art.Files
+
+				// Check if the package is already present
+				if p, err := systemDB.FindPackage(art.CompileSpec.GetPackage()); err == nil && p.GetName() != "" {
+					util.DefaultContext.Fatal("Package", art.CompileSpec.GetPackage().HumanReadableString(),
+						" already present.")
+				}
+
+				if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil {
+					util.DefaultContext.Fatal("Failed to create ", a, ": ", err.Error())
+				}
+				if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil {
+					util.DefaultContext.Fatal("Failed setting package files for ", a, ": ", err.Error())
+				}
+
+				util.DefaultContext.Info(art.CompileSpec.GetPackage().HumanReadableString(), " created")
+			}
+
+		},
+	}
+
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
+
+	return ans
+}

cmd/database/get.go

@@ -0,0 +1,85 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	"fmt"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"gopkg.in/yaml.v2"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseGetCommand() *cobra.Command {
+	var c = &cobra.Command{
+		Use:   "get <package>",
+		Short: "Get a package in the system DB as yaml",
+		Long: `Get a package in the system database in the YAML format:
+
+		$ luet database get system/foo
+
+To return also files:
+		$ luet database get --files system/foo`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			util.BindSystemFlags(cmd)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			showFiles, _ := cmd.Flags().GetBool("files")
+			util.SetSystemConfig(util.DefaultContext)
+
+			systemDB := util.DefaultContext.Config.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					continue
+				}
+
+				ps, err := systemDB.FindPackages(pack)
+				if err != nil {
+					continue
+				}
+				for _, p := range ps {
+					y, err := p.Yaml()
+					if err != nil {
+						continue
+					}
+					fmt.Println(string(y))
+					if showFiles {
+						files, err := systemDB.GetPackageFiles(p)
+						if err != nil {
+							continue
+						}
+						b, err := yaml.Marshal(files)
+						if err != nil {
+							continue
+						}
+						fmt.Println("files:\n" + string(b))
+					}
+				}
+			}
+		},
+	}
+	c.Flags().Bool("files", false, "Show package files.")
+	c.Flags().String("system-dbpath", "", "System db path")
+	c.Flags().String("system-target", "", "System rootpath")
+	c.Flags().String("system-engine", "", "System DB engine")
+
+	return c
+}

cmd/database/remove.go

@@ -0,0 +1,66 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_database
+
+import (
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+)
+
+func NewDatabaseRemoveCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "remove [package1] [package2] ...",
+		Short: "Remove a package from the system DB (forcefully - you normally don't want to do that)",
+		Long: `Removes a package in the system database without actually uninstalling it:
+
+		$ luet database remove foo/bar
+
+This commands takes multiple packages as arguments and prunes their entries from the system database.
+`,
+		Args: cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			util.BindSystemFlags(cmd)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			util.SetSystemConfig(util.DefaultContext)
+
+			systemDB := util.DefaultContext.Config.GetSystemDB()
+
+			for _, a := range args {
+				pack, err := helpers.ParsePackageStr(a)
+				if err != nil {
+					util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackage(pack); err != nil {
+					util.DefaultContext.Fatal("Failed removing ", a, ": ", err.Error())
+				}
+
+				if err := systemDB.RemovePackageFiles(pack); err != nil {
+					util.DefaultContext.Fatal("Failed removing files for ", a, ": ", err.Error())
+				}
+			}
+
+		},
+	}
+	ans.Flags().String("system-dbpath", "", "System db path")
+	ans.Flags().String("system-target", "", "System rootpath")
+	ans.Flags().String("system-engine", "", "System DB engine")
+
+	return ans
+}

cmd/exec.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/box"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var execCmd = &cobra.Command{
+	Use:   "exec --rootfs /path [command]",
+	Short: "Execute a command in the rootfs context",
+	Long:  `Uses unshare technique and pivot root to execute a command inside a folder containing a valid rootfs`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+	},
+	// If you change this, look at pkg/box/exec that runs this command and adapt
+	Run: func(cmd *cobra.Command, args []string) {
+
+		stdin, _ := cmd.Flags().GetBool("stdin")
+		stdout, _ := cmd.Flags().GetBool("stdout")
+		stderr, _ := cmd.Flags().GetBool("stderr")
+		rootfs, _ := cmd.Flags().GetString("rootfs")
+		base, _ := cmd.Flags().GetBool("decode")
+
+		entrypoint, _ := cmd.Flags().GetString("entrypoint")
+		envs, _ := cmd.Flags().GetStringArray("env")
+		mounts, _ := cmd.Flags().GetStringArray("mount")
+
+		if base {
+			var ss []string
+			for _, a := range args {
+				sDec, _ := b64.StdEncoding.DecodeString(a)
+				ss = append(ss, string(sDec))
+			}
+			//If the command to run is complex,using base64 to avoid bad input
+
+			args = ss
+		}
+		util.DefaultContext.Info("Executing", args, "in", rootfs)
+
+		b := box.NewBox(entrypoint, args, mounts, envs, rootfs, stdin, stdout, stderr)
+		err := b.Exec()
+		if err != nil {
+			util.DefaultContext.Fatal(errors.Wrap(err, fmt.Sprintf("entrypoint: %s rootfs: %s", entrypoint, rootfs)))
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	execCmd.Hidden = true
+	execCmd.Flags().String("rootfs", path, "Rootfs path")
+	execCmd.Flags().Bool("stdin", false, "Attach to stdin")
+	execCmd.Flags().Bool("stdout", false, "Attach to stdout")
+	execCmd.Flags().Bool("stderr", false, "Attach to stderr")
+	execCmd.Flags().Bool("decode", false, "Base64 decode")
+
+	execCmd.Flags().StringArrayP("env", "e", []string{}, "Environment settings")
+	execCmd.Flags().StringArrayP("mount", "m", []string{}, "List of paths to bind-mount from the host")
+
+	execCmd.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	RootCmd.AddCommand(execCmd)
+}

cmd/helpers/cli.go

@@ -0,0 +1,130 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	_gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo"
+	"github.com/mudler/luet/cmd/util"
+	pkg "github.com/mudler/luet/pkg/package"
+)
+
+func CreateRegexArray(rgx []string) ([]*regexp.Regexp, error) {
+	ans := make([]*regexp.Regexp, len(rgx))
+	if len(rgx) > 0 {
+		for idx, reg := range rgx {
+			re := regexp.MustCompile(reg)
+			if re == nil {
+				return nil, errors.New("Invalid regex " + reg + "!")
+			}
+			ans[idx] = re
+		}
+	}
+
+	return ans, nil
+}
+
+func packageData(p string) (string, string) {
+	cat := ""
+	name := ""
+	if strings.Contains(p, "/") {
+		packagedata := strings.Split(p, "/")
+		cat = packagedata[0]
+		name = packagedata[1]
+	} else {
+		name = p
+	}
+	return cat, name
+}
+
+func packageHasGentooSelector(v string) bool {
+	return (strings.HasPrefix(v, "=") || strings.HasPrefix(v, ">") ||
+		strings.HasPrefix(v, "<"))
+}
+
+func gentooVersion(gp *_gentoo.GentooPackage) string {
+
+	condition := gp.Condition.String()
+	if condition == "=" {
+		condition = ""
+	}
+
+	pkgVersion := fmt.Sprintf("%s%s%s",
+		condition,
+		gp.Version,
+		gp.VersionSuffix,
+	)
+	if gp.VersionBuild != "" {
+		pkgVersion = fmt.Sprintf("%s%s%s+%s",
+			condition,
+			gp.Version,
+			gp.VersionSuffix,
+			gp.VersionBuild,
+		)
+	}
+	return pkgVersion
+}
+
+func ParsePackageStr(p string) (*pkg.DefaultPackage, error) {
+
+	if packageHasGentooSelector(p) {
+		gp, err := _gentoo.ParsePackageStr(p)
+		if err != nil {
+			return nil, err
+		}
+		if gp.Version == "" {
+			gp.Version = "0"
+			gp.Condition = _gentoo.PkgCondGreaterEqual
+		}
+
+		return &pkg.DefaultPackage{
+			Name:     gp.Name,
+			Category: gp.Category,
+			Version:  gentooVersion(gp),
+			Uri:      make([]string, 0),
+		}, nil
+	}
+
+	ver := ">=0"
+	cat := ""
+	name := ""
+
+	if strings.Contains(p, "@") {
+		packageinfo := strings.Split(p, "@")
+		ver = packageinfo[1]
+		cat, name = packageData(packageinfo[0])
+	} else {
+		cat, name = packageData(p)
+	}
+
+	return &pkg.DefaultPackage{
+		Name:     name,
+		Category: cat,
+		Version:  ver,
+		Uri:      make([]string, 0),
+	}, nil
+}
+
+func CheckErr(err error) {
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+}

cmd/helpers/cli_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CLI helpers test Suite")
+}

cmd/helpers/cli_test.go

@@ -0,0 +1,102 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_helpers_test
+
+import (
+	. "github.com/mudler/luet/cmd/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CLI Helpers", func() {
+	Context("Can parse package strings correctly", func() {
+		It("accept single package names", func() {
+			pack, err := ParsePackageStr("foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal(""))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept unversioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept versioned packages with category", func() {
+			pack, err := ParsePackageStr("cat/foo@1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.1"))
+		})
+		It("accept versioned ranges with category", func() {
+			pack, err := ParsePackageStr("cat/foo@>=1.1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.1"))
+		})
+		It("accept gentoo regex parsing without versions", func() {
+			pack, err := ParsePackageStr("=cat/foo")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=0"))
+		})
+		It("accept gentoo regex parsing with versions", func() {
+			pack, err := ParsePackageStr("=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition", func() {
+			pack, err := ParsePackageStr(">=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">=1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition2", func() {
+			pack, err := ParsePackageStr("<cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition3", func() {
+			pack, err := ParsePackageStr(">cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal(">1.2"))
+		})
+
+		It("accept gentoo regex parsing with with condition4", func() {
+			pack, err := ParsePackageStr("<=cat/foo-1.2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(pack.GetName()).To(Equal("foo"))
+			Expect(pack.GetCategory()).To(Equal("cat"))
+			Expect(pack.GetVersion()).To(Equal("<=1.2"))
+		})
+	})
+})

cmd/install.go

@@ -0,0 +1,136 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var installCmd = &cobra.Command{
+	Use:   "install <pkg1> <pkg2> ...",
+	Short: "Install a package",
+	Long: `Installs one or more packages without asking questions:
+
+	$ luet install -y utils/busybox utils/yq ...
+	
+To install only deps of a package:
+	
+	$ luet install --onlydeps utils/busybox ...
+	
+To not install deps of a package:
+	
+	$ luet install --nodeps utils/busybox ...
+
+To force install a package:
+	
+	$ luet install --force utils/busybox ...
+`,
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toInstall pkg.Packages
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toInstall = append(toInstall, pack)
+		}
+
+		force := viper.GetBool("force")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		finalizerEnvs, _ := cmd.Flags().GetStringArray("finalizer-env")
+		relax, _ := cmd.Flags().GetBool("relax")
+
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		// Load finalizer runtime environments
+		err := util.SetCliFinalizerEnvs(util.DefaultContext, finalizerEnvs)
+		if err != nil {
+			util.DefaultContext.Fatal(err.Error())
+		}
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			DownloadOnly:                downloadOnly,
+			Ask:                         !yes,
+			Relaxed:                     relax,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err = inst.Install(toInstall, system)
+		if err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	installCmd.Flags().String("system-dbpath", "", "System db path")
+	installCmd.Flags().String("system-target", "", "System rootpath")
+	installCmd.Flags().String("system-engine", "", "System DB engine")
+
+	installCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	installCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	installCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	installCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	installCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	installCmd.Flags().Bool("relax", false, "Relax installation constraints")
+
+	installCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	installCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	installCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	installCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	installCmd.Flags().Bool("download-only", false, "Download only")
+	installCmd.Flags().StringArray("finalizer-env", []string{},
+		"Set finalizer environment in the format key=value.")
+
+	RootCmd.AddCommand(installCmd)
+}

cmd/pack.go

@@ -0,0 +1,99 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var packCmd = &cobra.Command{
+	Use:   "pack <package name>",
+	Short: "pack a custom package",
+	Long: `Pack creates a package from a directory, generating the metadata required from a tree to generate a repository.
+
+Pack can be used to manually replace what "luet build" does automatically by reading the packages build.yaml files.
+
+	$ mkdir -p output/etc/foo
+	$ echo "my config" > output/etc/foo
+	$ luet pack foo/bar@1.1 --source output
+
+Afterwards, you can use the content generated and associate it with a tree and a corresponding definition.yaml file with "luet create-repo".
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
+		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
+		viper.BindPFlag("source", cmd.Flags().Lookup("source"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		sourcePath := viper.GetString("source")
+
+		dst := viper.GetString("destination")
+		compressionType := viper.GetString("compression")
+		concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
+
+		if len(args) != 1 {
+			util.DefaultContext.Fatal("You must specify a package name")
+		}
+
+		packageName := args[0]
+
+		p, err := helpers.ParsePackageStr(packageName)
+		if err != nil {
+			util.DefaultContext.Fatal("Invalid package string ", packageName, ": ", err.Error())
+		}
+
+		spec := &compilerspec.LuetCompilationSpec{Package: p}
+		a := artifact.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
+		a.CompressionType = compression.Implementation(compressionType)
+		err = a.Compress(sourcePath, concurrency)
+		if err != nil {
+			util.DefaultContext.Fatal("failed compressing ", packageName, ": ", err.Error())
+		}
+		a.CompileSpec = spec
+		filelist, err := a.FileList()
+		if err != nil {
+			util.DefaultContext.Fatal("failed generating file list for ", packageName, ": ", err.Error())
+		}
+		a.Files = filelist
+		a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
+		err = a.WriteYAML(dst)
+		if err != nil {
+			util.DefaultContext.Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	packCmd.Flags().String("source", path, "Source folder")
+	packCmd.Flags().String("destination", path, "Destination folder")
+	packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
+
+	RootCmd.AddCommand(packCmd)
+}

cmd/query.go

@@ -1,112 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-package cmd
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log"
-
-	. "github.com/mudler/luet/pkg/logger"
-
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-	tree "github.com/mudler/luet/pkg/tree"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var queryCmd = &cobra.Command{
-	Use:   "query install <pkg>",
-	Short: "query other package manager tree into luet",
-	Long:  `Parses external PM and produces a luet parsable tree`,
-	Run: func(cmd *cobra.Command, args []string) {
-
-		input := viper.GetString("input")
-
-		if len(args) != 4 {
-			log.Fatalln("Incorrect number of arguments")
-		}
-		databaseType := viper.GetString("database")
-		var db pkg.PackageDatabase
-
-		switch databaseType {
-		case "memory":
-			db = pkg.NewInMemoryDatabase(false)
-		case "boltdb":
-			tmpdir, err := ioutil.TempDir("", "package")
-			if err != nil {
-				Fatal(err)
-			}
-			db = pkg.NewBoltDatabase(tmpdir)
-		}
-		defer db.Clean()
-
-		generalRecipe := tree.NewGeneralRecipe(db)
-		fmt.Println("Loading generated tree from " + input)
-
-		err := generalRecipe.Load(input)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
-
-		defer generalRecipe.Tree().GetPackageSet().Clean()
-
-		t := args[0]
-		v := args[1]
-		version := args[2]
-		cat := args[3]
-		switch t {
-		case "install":
-			// XXX: pack needs to be the same which is present in world.
-			// Tree caches generated world when using FindPackage
-			pack, err := generalRecipe.Tree().FindPackage(&pkg.DefaultPackage{Category: cat, Name: v, Version: version})
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-
-			fmt.Println("Install query from " + input + " [" + v + "]")
-			world, err := generalRecipe.Tree().World()
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			fmt.Println(">>> World")
-			for _, packss := range world {
-				packss.Explain()
-			}
-			s := solver.NewSolver([]pkg.Package{}, world, generalRecipe.Tree().GetPackageSet())
-			solution, err := s.Install([]pkg.Package{pack})
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
-			fmt.Println(">>> Solution")
-
-			for _, assertion := range solution {
-				assertion.Explain()
-			}
-		}
-
-	},
-}
-
-func init() {
-	queryCmd.Flags().String("input", "", "source folder")
-	viper.BindPFlag("input", queryCmd.Flags().Lookup("input"))
-	queryCmd.Flags().String("database", "memory", "database used for solving (memory,boltdb)")
-	viper.BindPFlag("database", queryCmd.Flags().Lookup("database"))
-
-	RootCmd.AddCommand(queryCmd)
-}

cmd/reclaim.go

@@ -0,0 +1,70 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/cmd/util"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var reclaimCmd = &cobra.Command{
+	Use:   "reclaim",
+	Short: "Reclaim packages to Luet database from available repositories",
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+	},
+	Long: `Reclaim tries to find association between packages in the online repositories and the system one.
+
+	$ luet reclaim
+
+It scans the target file system, and if finds a match with a package available in the repositories, it marks as installed in the system database.
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		util.SetSystemConfig(util.DefaultContext)
+
+		force := viper.GetBool("force")
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			Force:                       force,
+			PreserveSystemEssentialData: true,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err := inst.Reclaim(system)
+		if err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reclaimCmd.Flags().String("system-dbpath", "", "System db path")
+	reclaimCmd.Flags().String("system-target", "", "System rootpath")
+	reclaimCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reclaimCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+
+	RootCmd.AddCommand(reclaimCmd)
+}

cmd/reinstall.go

@@ -0,0 +1,114 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var reinstallCmd = &cobra.Command{
+	Use:   "reinstall <pkg1> <pkg2> <pkg3>",
+	Short: "reinstall a set of packages",
+	Long: `Reinstall a group of packages in the system:
+
+	$ luet reinstall -y system/busybox shells/bash system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig(util.DefaultContext)
+
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+			toAdd = append(toAdd, pack)
+		}
+
+		util.SetSolverConfig(util.DefaultContext)
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			NoDeps:                      true,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+			Context:                     util.DefaultContext,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	reinstallCmd.Flags().String("system-dbpath", "", "System db path")
+	reinstallCmd.Flags().String("system-target", "", "System rootpath")
+	reinstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	reinstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	reinstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	reinstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	reinstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	reinstallCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	reinstallCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	reinstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	reinstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	reinstallCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(reinstallCmd)
+}

cmd/replace.go

@@ -0,0 +1,125 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+
+	pkg "github.com/mudler/luet/pkg/package"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var replaceCmd = &cobra.Command{
+	Use:     "replace <pkg1> <pkg2> --for <pkg3> --for <pkg4> ...",
+	Short:   "replace a set of packages",
+	Aliases: []string{"r"},
+	Long: `Replaces one or a group of packages without asking questions:
+
+	$ luet replace -y system/busybox ... --for shells/bash --for system/coreutils ...
+`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("for", cmd.Flags().Lookup("for"))
+
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var toUninstall pkg.Packages
+		var toAdd pkg.Packages
+
+		f := viper.GetStringSlice("for")
+		force := viper.GetBool("force")
+		nodeps := viper.GetBool("nodeps")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
+		for _, a := range args {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toUninstall = append(toUninstall, pack)
+		}
+
+		for _, a := range f {
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toAdd = append(toAdd, pack)
+		}
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			OnlyDeps:                    onlydeps,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		err := inst.Swap(toUninstall, toAdd, system)
+		if err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	replaceCmd.Flags().String("system-dbpath", "", "System db path")
+	replaceCmd.Flags().String("system-target", "", "System rootpath")
+	replaceCmd.Flags().String("system-engine", "", "System DB engine")
+
+	replaceCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	replaceCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	replaceCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	replaceCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	replaceCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful!)")
+	replaceCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	replaceCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	replaceCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	replaceCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	replaceCmd.Flags().StringSlice("for", []string{}, "Packages that has to be installed in place of others")
+	replaceCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(replaceCmd)
+}

cmd/repo.go

@@ -0,0 +1,37 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/repo"
+
+	"github.com/spf13/cobra"
+)
+
+var repoGroupCmd = &cobra.Command{
+	Use:   "repo [command] [OPTIONS]",
+	Short: "Manage repositories",
+}
+
+func init() {
+	RootCmd.AddCommand(repoGroupCmd)
+
+	repoGroupCmd.AddCommand(
+		NewRepoListCommand(),
+		NewRepoUpdateCommand(),
+	)
+}

cmd/repo/list.go

@@ -0,0 +1,101 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/mudler/luet/cmd/util"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/pterm/pterm"
+
+	"github.com/spf13/cobra"
+)
+
+func NewRepoListCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "list [OPTIONS]",
+		Short: "List of the configured repositories.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var repoColor, repoText, repoRevision string
+
+			enable, _ := cmd.Flags().GetBool("enabled")
+			quiet, _ := cmd.Flags().GetBool("quiet")
+			repoType, _ := cmd.Flags().GetString("type")
+
+			for _, repo := range util.DefaultContext.Config.SystemRepositories {
+				if enable && !repo.Enable {
+					continue
+				}
+
+				if repoType != "" && repo.Type != repoType {
+					continue
+				}
+
+				repoRevision = ""
+
+				if quiet {
+					fmt.Println(repo.Name)
+				} else {
+					if repo.Enable {
+						repoColor = pterm.LightGreen(repo.Name)
+					} else {
+						repoColor = pterm.LightRed(repo.Name)
+					}
+					if repo.Description != "" {
+						repoText = pterm.LightYellow(repo.Description)
+					} else {
+						repoText = pterm.LightYellow(repo.Urls[0])
+					}
+
+					repobasedir := util.DefaultContext.Config.GetSystem().GetRepoDatabaseDirPath(repo.Name)
+					if repo.Cached {
+
+						r := installer.NewSystemRepository(repo)
+						localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir,
+							installer.REPOSITORY_SPECFILE))
+						if localRepo != nil {
+							tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64)
+							repoRevision = pterm.LightRed(localRepo.GetRevision()) +
+								" - " + pterm.LightGreen(time.Unix(tsec, 0).String())
+						}
+					}
+
+					if repoRevision != "" {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s\n  Revision %s", repoColor, repoText, repoRevision))
+					} else {
+						fmt.Println(
+							fmt.Sprintf("%s\n  %s", repoColor, repoText))
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().Bool("enabled", false, "Show only enable repositories.")
+	ans.Flags().BoolP("quiet", "q", false, "Show only name of the repositories.")
+	ans.Flags().StringP("type", "t", "", "Filter repositories of a specific type")
+
+	return ans
+}

cmd/repo/update.go

@@ -0,0 +1,79 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_repo
+
+import (
+	"github.com/mudler/luet/cmd/util"
+	installer "github.com/mudler/luet/pkg/installer"
+
+	"github.com/spf13/cobra"
+)
+
+func NewRepoUpdateCommand() *cobra.Command {
+	var ans = &cobra.Command{
+		Use:   "update [repo1] [repo2] [OPTIONS]",
+		Short: "Update a specific cached repository or all cached repositories.",
+		Example: `
+# Update all cached repositories:
+$> luet repo update
+
+# Update only repo1 and repo2
+$> luet repo update repo1 repo2
+`,
+		Aliases: []string{"up"},
+		PreRun: func(cmd *cobra.Command, args []string) {
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			ignore, _ := cmd.Flags().GetBool("ignore-errors")
+			force, _ := cmd.Flags().GetBool("force")
+
+			if len(args) > 0 {
+				for _, rname := range args {
+					repo, err := util.DefaultContext.Config.GetSystemRepository(rname)
+					if err != nil && !ignore {
+						util.DefaultContext.Fatal(err.Error())
+					} else if err != nil {
+						continue
+					}
+
+					r := installer.NewSystemRepository(*repo)
+					_, err = r.Sync(util.DefaultContext, force)
+					if err != nil && !ignore {
+						util.DefaultContext.Fatal("Error on sync repository " + rname + ": " + err.Error())
+					}
+				}
+
+			} else {
+				for _, repo := range util.DefaultContext.Config.SystemRepositories {
+					if repo.Cached && repo.Enable {
+						r := installer.NewSystemRepository(repo)
+						_, err := r.Sync(util.DefaultContext, force)
+						if err != nil && !ignore {
+							util.DefaultContext.Fatal("Error on sync repository " + r.GetName() + ": " + err.Error())
+						}
+					}
+				}
+			}
+		},
+	}
+
+	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	ans.Flags().BoolP("force", "f", false, "Force resync.")
+
+	return ans
+}

cmd/root.go

@@ -16,11 +16,11 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
-	"path"
-	"path/filepath"
 
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/cmd/util"
+	bus "github.com/mudler/luet/pkg/bus"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -29,59 +29,105 @@ import (
 var cfgFile string
 var Verbose bool
 
-const LuetCLIVersion = "0.1"
+const (
+	LuetCLIVersion = "0.19.2"
+	LuetEnvPrefix  = "LUET"
+)
+
+var license = []string{
+	"Luet Copyright (C) 2019-2021 Ettore Di Giacinto",
+	"This program comes with ABSOLUTELY NO WARRANTY.",
+	"This is free software, and you are welcome to redistribute it under certain conditions.",
+}
+
+// Build time and commit information.
+//
+// ⚠️ WARNING: should only be set by "-ldflags".
+var (
+	BuildTime   string
+	BuildCommit string
+)
+
+func version() string {
+	return fmt.Sprintf("%s-g%s %s", LuetCLIVersion, BuildCommit, BuildTime)
+}
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
-	Use:     "luet",
-	Short:   "Package manager for the XXth century!",
-	Long:    `Package manager which uses containers to build packages`,
-	Version: LuetCLIVersion,
+	Use:   "luet",
+	Short: "Container based package manager",
+	Long: `Luet is a single-binary package manager based on containers to build packages.
+	
+To install a package:
+
+	$ luet install package
+
+To search for a package in the repositories:
+
+$ luet search package
+
+To list all packages installed in the system:
+
+	$ luet search --installed .
+
+To show hidden packages:
+
+	$ luet search --hidden package
+
+To build a package, from a tree definition:
+
+	$ luet build --tree tree/path package
+	
+`,
+	Version: version(),
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		err := util.InitContext(util.DefaultContext)
+		if err != nil {
+			util.DefaultContext.Error("failed to load configuration:", err.Error())
+		}
+		util.DisplayVersionBanner(util.DefaultContext, util.IntroScreen, version, license)
+
+		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
+		// directly on sub command to ensure the creation only when it's
+		// needed.
+		err = util.DefaultContext.Config.GetSystem().InitTmpDir()
+		if err != nil {
+			util.DefaultContext.Fatal("failed on init tmp basedir:", err.Error())
+		}
+
+		viper.BindPFlag("plugin", cmd.Flags().Lookup("plugin"))
+
+		plugin := viper.GetStringSlice("plugin")
+
+		bus.Manager.Initialize(plugin...)
+		if len(bus.Manager.Plugins) != 0 {
+			util.DefaultContext.Info(":lollipop:Enabled plugins:")
+			for _, p := range bus.Manager.Plugins {
+				util.DefaultContext.Info("\t:arrow_right:", p.Name)
+			}
+		}
+	},
+	PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		// Cleanup all tmp directories used by luet
+		err := util.DefaultContext.Config.GetSystem().CleanupTmpDir()
+		if err != nil {
+			util.DefaultContext.Warning("failed on cleanup tmpdir:", err.Error())
+		}
+	},
+	SilenceErrors: true,
 }
 
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	util.HandleLock(util.DefaultContext)
+
 	if err := RootCmd.Execute(); err != nil {
-		Error(err)
+		fmt.Println(err)
 		os.Exit(-1)
 	}
 }
 
 func init() {
-	cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
-}
-
-// initConfig reads in config file and ENV variables if set.
-func initConfig() {
-	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
-	if err != nil {
-		Error(err)
-		os.Exit(1)
-	}
-
-	viper.SetConfigName(".luet") // name of config file (without extension)
-	if cfgFile != "" {           // enable ability to specify config file via flag
-		Info(">>> cfgFile: ", cfgFile)
-		viper.SetConfigFile(cfgFile)
-		configDir := path.Dir(cfgFile)
-		if configDir != "." && configDir != dir {
-			viper.AddConfigPath(configDir)
-		}
-	}
-
-	viper.AddConfigPath(dir)
-	viper.AddConfigPath(".")
-	viper.AddConfigPath("$HOME")
-	viper.AutomaticEnv() // read in environment variables that match
-
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		Info("Using config file:", viper.ConfigFileUsed())
-	} else {
-		Warning(err)
-	}
-
+	util.InitViper(util.DefaultContext, RootCmd)
 }

cmd/search.go

@@ -0,0 +1,401 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+type PackageResult struct {
+	Name       string   `json:"name"`
+	Category   string   `json:"category"`
+	Version    string   `json:"version"`
+	Repository string   `json:"repository"`
+	Target     string   `json:"target"`
+	Hidden     bool     `json:"hidden"`
+	Files      []string `json:"files"`
+}
+
+type Results struct {
+	Packages []PackageResult `json:"packages"`
+}
+
+func (r PackageResult) String() string {
+	return fmt.Sprintf("%s/%s-%s required for %s", r.Category, r.Name, r.Version, r.Target)
+}
+
+var rows []string = []string{"Package", "Category", "Name", "Version", "Repository", "License"}
+
+func packageToRow(repo string, p pkg.Package) []string {
+	return []string{p.HumanReadableString(), p.GetCategory(), p.GetName(), p.GetVersion(), repo, p.GetLicense()}
+}
+
+func packageToList(l *util.ListWriter, repo string, p pkg.Package) {
+	l.AppendItem(pterm.BulletListItem{
+		Level: 0, Text: p.HumanReadableString(),
+		TextStyle: pterm.NewStyle(pterm.FgCyan), Bullet: ">", BulletStyle: pterm.NewStyle(pterm.FgYellow),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Category: %s", p.GetCategory()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Name: %s", p.GetName()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Version: %s", p.GetVersion()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Description: %s", p.GetDescription()),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Repository: %s ", repo),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+	l.AppendItem(pterm.BulletListItem{
+		Level: 1, Text: fmt.Sprintf("Uri: %s ", strings.Join(p.GetURI(), " ")),
+		Bullet: "->", BulletStyle: pterm.NewStyle(pterm.FgDarkGray),
+	})
+}
+
+func searchLocally(term string, l *util.ListWriter, t *util.TableWriter, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+
+	var err error
+	iMatches := pkg.Packages{}
+	if label {
+		iMatches, err = system.Database.FindPackageLabel(term)
+	} else if labelMatch {
+		iMatches, err = system.Database.FindPackageLabelMatch(term)
+	} else {
+		iMatches, err = system.Database.FindPackageMatch(term)
+	}
+
+	if err != nil {
+		util.DefaultContext.Fatal("Error: " + err.Error())
+	}
+
+	for _, pack := range iMatches {
+		if !revdeps {
+			if !pack.IsHidden() || pack.IsHidden() && hidden {
+
+				t.AppendRow(packageToRow("system", pack))
+				packageToList(l, "system", pack)
+				f, _ := system.Database.GetPackageFiles(pack)
+				results.Packages = append(results.Packages,
+					PackageResult{
+						Name:       pack.GetName(),
+						Version:    pack.GetVersion(),
+						Category:   pack.GetCategory(),
+						Repository: "system",
+						Hidden:     pack.IsHidden(),
+						Files:      f,
+					})
+			}
+		} else {
+
+			packs, _ := system.Database.GetRevdeps(pack)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow("system", pack))
+					packageToList(l, "system", pack)
+					f, _ := system.Database.GetPackageFiles(revdep)
+					results.Packages = append(results.Packages,
+						PackageResult{
+							Name:       revdep.GetName(),
+							Version:    revdep.GetVersion(),
+							Category:   revdep.GetCategory(),
+							Repository: "system",
+							Hidden:     revdep.IsHidden(),
+							Files:      f,
+						})
+				}
+			}
+		}
+	}
+	return results
+
+}
+func searchOnline(term string, l *util.ListWriter, t *util.TableWriter, label, labelMatch, revdeps, hidden bool) Results {
+	var results Results
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
+			Context:             util.DefaultContext,
+		},
+	)
+	synced, err := inst.SyncRepositories()
+	if err != nil {
+		util.DefaultContext.Fatal("Error: " + err.Error())
+	}
+
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+	if label {
+		matches = synced.SearchLabel(term)
+	} else if labelMatch {
+		matches = synced.SearchLabelMatch(term)
+	} else {
+		matches = synced.Search(term)
+	}
+
+	for _, m := range matches {
+		if !revdeps {
+			if !m.Package.IsHidden() || m.Package.IsHidden() && hidden {
+				t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+				packageToList(l, m.Repo.GetName(), m.Package)
+				r := &PackageResult{
+					Name:       m.Package.GetName(),
+					Version:    m.Package.GetVersion(),
+					Category:   m.Package.GetCategory(),
+					Repository: m.Repo.GetName(),
+					Hidden:     m.Package.IsHidden(),
+				}
+				if m.Artifact != nil {
+					r.Files = m.Artifact.Files
+				}
+				results.Packages = append(results.Packages, *r)
+			}
+		} else {
+			packs, _ := m.Repo.GetTree().GetDatabase().GetRevdeps(m.Package)
+			for _, revdep := range packs {
+				if !revdep.IsHidden() || revdep.IsHidden() && hidden {
+					t.AppendRow(packageToRow(m.Repo.GetName(), revdep))
+					packageToList(l, m.Repo.GetName(), revdep)
+					r := &PackageResult{
+						Name:       revdep.GetName(),
+						Version:    revdep.GetVersion(),
+						Category:   revdep.GetCategory(),
+						Repository: m.Repo.GetName(),
+						Hidden:     revdep.IsHidden(),
+					}
+					if m.Artifact != nil {
+						r.Files = m.Artifact.Files
+					}
+					results.Packages = append(results.Packages, *r)
+				}
+			}
+		}
+	}
+	return results
+}
+func searchLocalFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
+	var results Results
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
+
+	matches, _ := util.DefaultContext.Config.GetSystemDB().FindPackageByFile(term)
+	for _, pack := range matches {
+		t.AppendRow(packageToRow("system", pack))
+		packageToList(l, "system", pack)
+		f, _ := util.DefaultContext.Config.GetSystemDB().GetPackageFiles(pack)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       pack.GetName(),
+				Version:    pack.GetVersion(),
+				Category:   pack.GetCategory(),
+				Repository: "system",
+				Hidden:     pack.IsHidden(),
+				Files:      f,
+			})
+	}
+
+	return results
+}
+
+func searchFiles(term string, l *util.ListWriter, t *util.TableWriter) Results {
+	var results Results
+
+	inst := installer.NewLuetInstaller(
+		installer.LuetInstallerOptions{
+			Concurrency:         util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:       *util.DefaultContext.Config.GetSolverOptions(),
+			PackageRepositories: util.DefaultContext.Config.SystemRepositories,
+			Context:             util.DefaultContext,
+		},
+	)
+	synced, err := inst.SyncRepositories()
+	if err != nil {
+		util.DefaultContext.Fatal("Error: " + err.Error())
+	}
+
+	util.DefaultContext.Info("--- Search results (" + term + "): ---")
+
+	matches := []installer.PackageMatch{}
+
+	matches = synced.SearchPackages(term, installer.FileSearch)
+
+	for _, m := range matches {
+		t.AppendRow(packageToRow(m.Repo.GetName(), m.Package))
+		packageToList(l, m.Repo.GetName(), m.Package)
+		results.Packages = append(results.Packages,
+			PackageResult{
+				Name:       m.Package.GetName(),
+				Version:    m.Package.GetVersion(),
+				Category:   m.Package.GetCategory(),
+				Repository: m.Repo.GetName(),
+				Hidden:     m.Package.IsHidden(),
+				Files:      m.Artifact.Files,
+			})
+	}
+	return results
+}
+
+var searchCmd = &cobra.Command{
+	Use:   "search <term>",
+	Short: "Search packages",
+	Long: `Search for installed and available packages
+	
+To search a package in the repositories:
+
+	$ luet search <regex>
+
+To search a package and display results in a table (wide screens):
+
+	$ luet search --table <regex>
+
+To look into the installed packages:
+
+	$ luet search --installed <regex>
+
+Note: the regex argument is optional, if omitted implies "all"
+
+To search a package by label:
+
+	$ luet search --by-label <label>
+
+or by regex against the label:
+
+	$ luet search --by-label-regex <label>
+
+It can also show a package revdeps by:
+
+	$ luet search --revdeps <regex>
+
+Search can also return results in the terminal in different ways: as terminal output, as json or as yaml.
+
+	$ luet search --json <regex> # JSON output
+	$ luet search --yaml <regex> # YAML output
+`,
+	Aliases: []string{"s"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("installed", cmd.Flags().Lookup("installed"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		var results Results
+		if len(args) > 1 {
+			util.DefaultContext.Fatal("Wrong number of arguments (expected 1)")
+		} else if len(args) == 0 {
+			args = []string{"."}
+		}
+		hidden, _ := cmd.Flags().GetBool("hidden")
+
+		installed := viper.GetBool("installed")
+		searchWithLabel, _ := cmd.Flags().GetBool("by-label")
+		searchWithLabelMatch, _ := cmd.Flags().GetBool("by-label-regex")
+		revdeps, _ := cmd.Flags().GetBool("revdeps")
+		tableMode, _ := cmd.Flags().GetBool("table")
+		files, _ := cmd.Flags().GetBool("files")
+
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
+
+		out, _ := cmd.Flags().GetString("output")
+		if out != "terminal" {
+			util.DefaultContext.Config.GetLogging().SetLogLevel("error")
+		}
+
+		l := &util.ListWriter{}
+		t := &util.TableWriter{}
+		t.AppendRow(rows)
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		switch {
+		case files && installed:
+			results = searchLocalFiles(args[0], l, t)
+		case files && !installed:
+			results = searchFiles(args[0], l, t)
+		case !installed:
+			results = searchOnline(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
+		default:
+			results = searchLocally(args[0], l, t, searchWithLabel, searchWithLabelMatch, revdeps, hidden)
+		}
+
+		if tableMode {
+			t.Render()
+		} else {
+			l.Render()
+		}
+
+		y, err := yaml.Marshal(results)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+			return
+		}
+		switch out {
+		case "yaml":
+			fmt.Println(string(y))
+		case "json":
+			j2, err := yaml.YAMLToJSON(y)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			fmt.Println(string(j2))
+		}
+
+	},
+}
+
+func init() {
+	searchCmd.Flags().String("system-dbpath", "", "System db path")
+	searchCmd.Flags().String("system-target", "", "System rootpath")
+	searchCmd.Flags().String("system-engine", "", "System DB engine")
+
+	searchCmd.Flags().Bool("installed", false, "Search between system packages")
+	searchCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	searchCmd.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	searchCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	searchCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	searchCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	searchCmd.Flags().Bool("by-label", false, "Search packages through label")
+	searchCmd.Flags().Bool("by-label-regex", false, "Search packages through label regex")
+	searchCmd.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	searchCmd.Flags().Bool("hidden", false, "Include hidden packages")
+	searchCmd.Flags().Bool("table", false, "show output in a table (wider screens)")
+	searchCmd.Flags().Bool("files", false, "Search between packages files")
+
+	RootCmd.AddCommand(searchCmd)
+}

cmd/serve-repo.go

@@ -0,0 +1,59 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var serverepoCmd = &cobra.Command{
+	Use:   "serve-repo",
+	Short: "Embedded micro-http server",
+	Long:  `Embedded mini http server for serving local repositories`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("dir", cmd.Flags().Lookup("dir"))
+		viper.BindPFlag("address", cmd.Flags().Lookup("address"))
+		viper.BindPFlag("port", cmd.Flags().Lookup("port"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		dir := viper.GetString("dir")
+		port := viper.GetString("port")
+		address := viper.GetString("address")
+
+		http.Handle("/", http.FileServer(http.Dir(dir)))
+
+		util.DefaultContext.Info("Serving ", dir, " on HTTP port: ", port)
+		util.DefaultContext.Fatal(http.ListenAndServe(address+":"+port, nil))
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	serverepoCmd.Flags().String("dir", path, "Packages folder (output from build)")
+	serverepoCmd.Flags().String("port", "9090", "Listening port")
+	serverepoCmd.Flags().String("address", "0.0.0.0", "Listening address")
+
+	RootCmd.AddCommand(serverepoCmd)
+}

cmd/tree.go

@@ -0,0 +1,39 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	. "github.com/mudler/luet/cmd/tree"
+
+	"github.com/spf13/cobra"
+)
+
+var treeGroupCmd = &cobra.Command{
+	Use:   "tree [command] [OPTIONS]",
+	Short: "Tree operations",
+}
+
+func init() {
+	RootCmd.AddCommand(treeGroupCmd)
+
+	treeGroupCmd.AddCommand(
+		NewTreePkglistCommand(),
+		NewTreeValidateCommand(),
+		NewTreeBumpCommand(),
+		NewTreeImageCommand(),
+	)
+}

cmd/tree/bump.go

@@ -0,0 +1,88 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+
+	"github.com/mudler/luet/cmd/util"
+	spectooling "github.com/mudler/luet/pkg/spectooling"
+	tree "github.com/mudler/luet/pkg/tree"
+	version "github.com/mudler/luet/pkg/versioner"
+
+	"github.com/spf13/cobra"
+)
+
+func NewTreeBumpCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "bump [OPTIONS]",
+		Short: "Bump a new package build version.",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			df, _ := cmd.Flags().GetString("definition-file")
+			if df == "" {
+				util.DefaultContext.Fatal("Mandatory definition.yaml path missing.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			spec, _ := cmd.Flags().GetString("definition-file")
+			toStdout, _ := cmd.Flags().GetBool("to-stdout")
+			pkgVersion, _ := cmd.Flags().GetString("pkg-version")
+			pack, err := tree.ReadDefinitionFile(spec)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+
+			if pkgVersion != "" {
+				validator := &version.WrappedVersioner{}
+				err := validator.Validate(pkgVersion)
+				if err != nil {
+					util.DefaultContext.Fatal("Invalid version string: " + err.Error())
+				}
+				pack.SetVersion(pkgVersion)
+			} else {
+				// Retrieve version build section with Gentoo parser
+				err = pack.BumpBuildVersion()
+				if err != nil {
+					util.DefaultContext.Fatal("Error on increment build version: " + err.Error())
+				}
+			}
+			if toStdout {
+				data, err := spectooling.NewDefaultPackageSanitized(&pack).Yaml()
+				if err != nil {
+					util.DefaultContext.Fatal("Error on yaml conversion: " + err.Error())
+				}
+				fmt.Println(string(data))
+			} else {
+
+				err = tree.WriteDefinitionFile(&pack, spec)
+				if err != nil {
+					util.DefaultContext.Fatal("Error on write definition file: " + err.Error())
+				}
+
+				fmt.Printf("Bumped package %s/%s-%s.\n", pack.Category, pack.Name, pack.Version)
+			}
+		},
+	}
+
+	ans.Flags().StringP("pkg-version", "p", "", "Set a specific package version")
+	ans.Flags().StringP("definition-file", "f", "", "Path of the definition to bump.")
+	ans.Flags().BoolP("to-stdout", "o", false, "Bump package to output.")
+
+	return ans
+}

cmd/tree/images.go

@@ -0,0 +1,155 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewTreeImageCommand() *cobra.Command {
+
+	var ans = &cobra.Command{
+		Use:   "images [OPTIONS]",
+		Short: "List of the images of a package",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
+			}
+
+			if len(args) != 1 {
+				util.DefaultContext.Fatal("Expects one package as parameter")
+			}
+			util.BindValuesFlags(cmd)
+			viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			imageRepository := viper.GetString("image-repository")
+			pullRepo, _ := cmd.Flags().GetStringArray("pull-repository")
+			values := util.ValuesFlags()
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
+			}
+
+			reciper := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					util.DefaultContext.Fatal("Error on load tree ", err)
+				}
+			}
+			compilerBackend := backend.NewSimpleDockerBackend(util.DefaultContext)
+
+			opts := *util.DefaultContext.Config.GetSolverOptions()
+			opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1}
+			luetCompiler := compiler.NewLuetCompiler(
+				compilerBackend,
+				reciper.GetDatabase(),
+				options.WithBuildValues(values),
+				options.WithContext(util.DefaultContext),
+				options.WithPushRepository(imageRepository),
+				options.WithPullRepositories(pullRepo),
+				options.WithTemplateFolder(util.TemplateFolders(util.DefaultContext, false, treePath)),
+				options.WithSolverOptions(opts),
+			)
+
+			a := args[0]
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+
+			spec, err := luetCompiler.FromPackage(pack)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+
+			ht := compiler.NewHashTree(reciper.GetDatabase())
+			hashtree, err := ht.Query(luetCompiler, spec)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+
+			for _, assertion := range hashtree.Solution { //highly dependent on the order
+
+				//buildImageHash := imageRepository + ":" + assertion.Hash.BuildHash
+				currentPackageImageHash := imageRepository + ":" + assertion.Hash.PackageHash
+
+				results.Packages = append(results.Packages, TreePackageResult{
+					Name:     assertion.Package.GetName(),
+					Version:  assertion.Package.GetVersion(),
+					Category: assertion.Package.GetCategory(),
+					Image:    currentPackageImageHash,
+				})
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				for _, p := range results.Packages {
+					fmt.Println(fmt.Sprintf("%s/%s-%s: %s", p.Category, p.Name, p.Version, p.Image))
+				}
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
+	ans.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
+	ans.Flags().StringArrayP("pull-repository", "p", []string{}, "A list of repositories to pull the cache from")
+
+	return ans
+}

cmd/tree/pkglist.go

@@ -0,0 +1,287 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/ghodss/yaml"
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type TreePackageResult struct {
+	Name     string `json:"name"`
+	Category string `json:"category"`
+	Version  string `json:"version"`
+	Path     string `json:"path"`
+	Image    string `json:"image"`
+}
+
+type TreeResults struct {
+	Packages []TreePackageResult `json:"packages"`
+}
+
+func pkgDetail(pkg pkg.Package) string {
+	ans := fmt.Sprintf(`
+  @@ Package: %s/%s-%s
+     Description: %s
+     License:     %s`,
+		pkg.GetCategory(), pkg.GetName(), pkg.GetVersion(),
+		pkg.GetDescription(), pkg.GetLicense())
+
+	for idx, u := range pkg.GetURI() {
+		if idx == 0 {
+			ans += fmt.Sprintf("     URLs:        %s", u)
+		} else {
+			ans += fmt.Sprintf("                  %s", u)
+		}
+	}
+
+	return ans
+}
+
+func NewTreePkglistCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+
+	var ans = &cobra.Command{
+		Use:   "pkglist [OPTIONS]",
+		Short: "List of the packages found in tree.",
+		Args:  cobra.NoArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			t, _ := cmd.Flags().GetStringArray("tree")
+			if len(t) == 0 {
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
+			}
+
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+			if revdeps && deps {
+				util.DefaultContext.Fatal("Both revdeps and deps option used. Choice only one.")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var results TreeResults
+			var depSolver solver.PackageSolver
+
+			treePath, _ := cmd.Flags().GetStringArray("tree")
+			verbose, _ := cmd.Flags().GetBool("verbose")
+			buildtime, _ := cmd.Flags().GetBool("buildtime")
+			full, _ := cmd.Flags().GetBool("full")
+			revdeps, _ := cmd.Flags().GetBool("revdeps")
+			deps, _ := cmd.Flags().GetBool("deps")
+
+			out, _ := cmd.Flags().GetString("output")
+			if out != "terminal" {
+				util.DefaultContext.Config.GetLogging().SetLogLevel("error")
+			}
+
+			var reciper tree.Builder
+			if buildtime {
+				reciper = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			} else {
+				reciper = tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))
+			}
+
+			for _, t := range treePath {
+				err := reciper.Load(t)
+				if err != nil {
+					util.DefaultContext.Fatal("Error on load tree ", err)
+				}
+			}
+
+			if deps {
+				emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+
+				depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+					reciper.GetDatabase(),
+					emptyInstallationDb)
+
+			}
+
+			regExcludes, err := helpers.CreateRegexArray(excludes)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+			regMatches, err := helpers.CreateRegexArray(matches)
+			if err != nil {
+				util.DefaultContext.Fatal(err.Error())
+			}
+
+			plist := make([]string, 0)
+			for _, p := range reciper.GetDatabase().World() {
+				pkgstr := ""
+				addPkg := true
+				if full {
+					pkgstr = pkgDetail(p)
+				} else if verbose {
+					pkgstr = p.HumanReadableString()
+				} else {
+					pkgstr = fmt.Sprintf("%s/%s", p.GetCategory(), p.GetName())
+				}
+
+				if len(matches) > 0 {
+					matched := false
+					for _, rgx := range regMatches {
+						if rgx.MatchString(pkgstr) {
+							matched = true
+							break
+						}
+					}
+					if !matched {
+						addPkg = false
+					}
+				}
+
+				if len(excludes) > 0 && addPkg {
+					for _, rgx := range regExcludes {
+						if rgx.MatchString(pkgstr) {
+							addPkg = false
+							break
+						}
+					}
+				}
+
+				if !addPkg {
+					continue
+				}
+
+				if revdeps {
+					packs, _ := reciper.GetDatabase().GetRevdeps(p)
+					for i := range packs {
+						revdep := packs[i]
+						if full {
+							pkgstr = pkgDetail(revdep)
+						} else if verbose {
+							pkgstr = revdep.HumanReadableString()
+						} else {
+							pkgstr = fmt.Sprintf("%s/%s", revdep.GetCategory(), revdep.GetName())
+						}
+						plist = append(plist, pkgstr)
+						results.Packages = append(results.Packages, TreePackageResult{
+							Name:     revdep.GetName(),
+							Version:  revdep.GetVersion(),
+							Category: revdep.GetCategory(),
+							Path:     revdep.GetPath(),
+						})
+					}
+				} else if deps {
+
+					solution, err := depSolver.Install(pkg.Packages{p})
+					if err != nil {
+						util.DefaultContext.Fatal(err.Error())
+					}
+					ass := solution.SearchByName(p.GetPackageName())
+					solution, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					if err != nil {
+						util.DefaultContext.Fatal(err.Error())
+					}
+
+					for _, pa := range solution {
+
+						if pa.Value {
+							// Exclude itself
+							if pa.Package.GetName() == p.GetName() && pa.Package.GetCategory() == p.GetCategory() {
+								continue
+							}
+
+							if full {
+								pkgstr = pkgDetail(pa.Package)
+							} else if verbose {
+								pkgstr = pa.Package.HumanReadableString()
+							} else {
+								pkgstr = fmt.Sprintf("%s/%s", pa.Package.GetCategory(), pa.Package.GetName())
+							}
+							plist = append(plist, pkgstr)
+							results.Packages = append(results.Packages, TreePackageResult{
+								Name:     pa.Package.GetName(),
+								Version:  pa.Package.GetVersion(),
+								Category: pa.Package.GetCategory(),
+								Path:     pa.Package.GetPath(),
+							})
+						}
+
+					}
+
+				} else {
+
+					plist = append(plist, pkgstr)
+					results.Packages = append(results.Packages, TreePackageResult{
+						Name:     p.GetName(),
+						Version:  p.GetVersion(),
+						Category: p.GetCategory(),
+						Path:     p.GetPath(),
+					})
+
+				}
+
+			}
+
+			y, err := yaml.Marshal(results)
+			if err != nil {
+				fmt.Printf("err: %v\n", err)
+				return
+			}
+			switch out {
+			case "yaml":
+				fmt.Println(string(y))
+			case "json":
+				j2, err := yaml.YAMLToJSON(y)
+				if err != nil {
+					fmt.Printf("err: %v\n", err)
+					return
+				}
+				fmt.Println(string(j2))
+			default:
+				if !deps {
+					sort.Strings(plist)
+				}
+				for _, p := range plist {
+					fmt.Println(p)
+				}
+			}
+
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	ans.Flags().BoolP("buildtime", "b", false, "Build time match")
+	ans.Flags().StringP("output", "o", "terminal", "Output format ( Defaults: terminal, available: json,yaml )")
+	ans.Flags().Bool("revdeps", false, "Search package reverse dependencies")
+	ans.Flags().Bool("deps", false, "Search package dependencies")
+
+	ans.Flags().BoolP("verbose", "v", false, "Add package version")
+	ans.Flags().BoolP("full", "f", false, "Show package detail")
+	ans.Flags().StringArrayP("tree", "t", []string{path}, "Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Include only matched packages from list. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from list. (Use string as regex).")
+
+	return ans
+}

cmd/tree/validate.go

@@ -0,0 +1,500 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd_tree
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"sync"
+
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	tree "github.com/mudler/luet/pkg/tree"
+
+	"github.com/spf13/cobra"
+)
+
+type ValidateOpts struct {
+	WithSolver    bool
+	OnlyRuntime   bool
+	OnlyBuildtime bool
+	RegExcludes   []*regexp.Regexp
+	RegMatches    []*regexp.Regexp
+	Excludes      []string
+	Matches       []string
+
+	// Runtime validate stuff
+	RuntimeCacheDeps *pkg.InMemoryDatabase
+	RuntimeReciper   *tree.InstallerRecipe
+
+	// Buildtime validate stuff
+	BuildtimeCacheDeps *pkg.InMemoryDatabase
+	BuildtimeReciper   *tree.CompilerRecipe
+
+	Mutex      sync.Mutex
+	BrokenPkgs int
+	BrokenDeps int
+
+	Errors []error
+}
+
+func (o *ValidateOpts) IncrBrokenPkgs() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenPkgs++
+}
+
+func (o *ValidateOpts) IncrBrokenDeps() {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.BrokenDeps++
+}
+
+func (o *ValidateOpts) AddError(err error) {
+	o.Mutex.Lock()
+	defer o.Mutex.Unlock()
+	o.Errors = append(o.Errors, err)
+}
+
+func validatePackage(p pkg.Package, checkType string, opts *ValidateOpts, reciper tree.Builder, cacheDeps *pkg.InMemoryDatabase) error {
+	var errstr string
+	var ans error
+
+	var depSolver solver.PackageSolver
+
+	if opts.WithSolver {
+		emptyInstallationDb := pkg.NewInMemoryDatabase(false)
+		depSolver = solver.NewSolver(solver.Options{Type: solver.SingleCoreSimple}, pkg.NewInMemoryDatabase(false),
+			reciper.GetDatabase(),
+			emptyInstallationDb)
+	}
+
+	found, err := reciper.GetDatabase().FindPackages(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  ">=0",
+		},
+	)
+
+	if err != nil || len(found) < 1 {
+		if err != nil {
+			errstr = err.Error()
+		} else {
+			errstr = "No packages"
+		}
+		util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			errstr,
+		))
+
+		opts.IncrBrokenDeps()
+
+		return errors.New(
+			fmt.Sprintf("[%9s] %s/%s-%s: Broken. No versions could be found by database %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				errstr,
+			))
+	}
+
+	// Ensure that we use the right package from right recipier for deps
+	pReciper, err := reciper.GetDatabase().FindPackage(
+		&pkg.DefaultPackage{
+			Name:     p.GetName(),
+			Category: p.GetCategory(),
+			Version:  p.GetVersion(),
+		},
+	)
+	if err != nil {
+		errstr = fmt.Sprintf("[%9s] %s/%s-%s: Error on retrieve package - %s.",
+			checkType,
+			p.GetCategory(), p.GetName(), p.GetVersion(),
+			err.Error(),
+		)
+		util.DefaultContext.Error(errstr)
+
+		return errors.New(errstr)
+	}
+	p = pReciper
+
+	pkgstr := fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(),
+		p.GetVersion())
+
+	validpkg := true
+
+	if len(opts.Matches) > 0 {
+		matched := false
+		for _, rgx := range opts.RegMatches {
+			if rgx.MatchString(pkgstr) {
+				matched = true
+				break
+			}
+		}
+
+		if !matched {
+			return nil
+		}
+	}
+
+	if len(opts.Excludes) > 0 {
+		excluded := false
+		for _, rgx := range opts.RegExcludes {
+			if rgx.MatchString(pkgstr) {
+				excluded = true
+				break
+			}
+		}
+
+		if excluded {
+			return nil
+		}
+	}
+
+	util.DefaultContext.Info(fmt.Sprintf("[%9s] Checking package ", checkType)+
+		fmt.Sprintf("%s/%s-%s", p.GetCategory(), p.GetName(), p.GetVersion()),
+		"with", len(p.GetRequires()), "dependencies and", len(p.GetConflicts()), "conflicts.")
+
+	all := p.GetRequires()
+	all = append(all, p.GetConflicts()...)
+	for idx, r := range all {
+
+		var deps pkg.Packages
+		var err error
+		if r.IsSelector() {
+			deps, err = reciper.GetDatabase().FindPackages(
+				&pkg.DefaultPackage{
+					Name:     r.GetName(),
+					Category: r.GetCategory(),
+					Version:  r.GetVersion(),
+				},
+			)
+		} else {
+			deps = append(deps, r)
+		}
+
+		if err != nil || len(deps) < 1 {
+			if err != nil {
+				errstr = err.Error()
+			} else {
+				errstr = "No packages"
+			}
+			util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+				checkType,
+				p.GetCategory(), p.GetName(), p.GetVersion(),
+				r.GetCategory(), r.GetName(), r.GetVersion(),
+				errstr,
+			))
+
+			opts.IncrBrokenDeps()
+
+			ans = errors.New(
+				fmt.Sprintf("[%9s] %s/%s-%s: Broken Dep %s/%s-%s - %s",
+					checkType,
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+					errstr))
+
+			validpkg = false
+
+		} else {
+
+			util.DefaultContext.Debug(fmt.Sprintf("[%9s] Find packages for dep", checkType),
+				fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+
+			if opts.WithSolver {
+
+				util.DefaultContext.Info(fmt.Sprintf("[%9s]  :soap: [%2d/%2d] %s/%s-%s: %s/%s-%s",
+					checkType,
+					idx+1, len(all),
+					p.GetCategory(), p.GetName(), p.GetVersion(),
+					r.GetCategory(), r.GetName(), r.GetVersion(),
+				))
+
+				// Check if the solver is already been done for the deep
+				_, err := cacheDeps.Get(r.HashFingerprint(""))
+				if err == nil {
+					util.DefaultContext.Debug(fmt.Sprintf("[%9s]  :direct_hit: Cache Hit for dep", checkType),
+						fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion()))
+					continue
+				}
+
+				util.DefaultContext.Spinner()
+				solution, err := depSolver.Install(pkg.Packages{r})
+				ass := solution.SearchByName(r.GetPackageName())
+				util.DefaultContext.SpinnerStop()
+				if err == nil {
+					if ass == nil {
+
+						ans = errors.New(
+							fmt.Sprintf("[%9s] %s/%s-%s: solution doesn't retrieve package %s/%s-%s.",
+								checkType,
+								p.GetCategory(), p.GetName(), p.GetVersion(),
+								r.GetCategory(), r.GetName(), r.GetVersion(),
+							))
+
+						if util.DefaultContext.Config.GetGeneral().Debug {
+							for idx, pa := range solution {
+								fmt.Println(fmt.Sprintf("[%9s] %s/%s-%s: solution %d: %s",
+									checkType,
+									p.GetCategory(), p.GetName(), p.GetVersion(), idx,
+									pa.Package.GetPackageName()))
+							}
+						}
+
+						util.DefaultContext.Error(ans.Error())
+						opts.IncrBrokenDeps()
+						validpkg = false
+					} else {
+						_, err = solution.Order(reciper.GetDatabase(), ass.Package.GetFingerPrint())
+					}
+				}
+
+				if err != nil {
+
+					util.DefaultContext.Error(fmt.Sprintf("[%9s] %s/%s-%s: solver broken for dep %s/%s-%s - %s",
+						checkType,
+						p.GetCategory(), p.GetName(), p.GetVersion(),
+						r.GetCategory(), r.GetName(), r.GetVersion(),
+						err.Error(),
+					))
+
+					ans = errors.New(
+						fmt.Sprintf("[%9s] %s/%s-%s: solver broken for Dep %s/%s-%s - %s",
+							checkType,
+							p.GetCategory(), p.GetName(), p.GetVersion(),
+							r.GetCategory(), r.GetName(), r.GetVersion(),
+							err.Error()))
+
+					opts.IncrBrokenDeps()
+					validpkg = false
+				}
+
+				// Register the key
+				cacheDeps.Set(r.HashFingerprint(""), "1")
+
+			}
+		}
+
+	}
+
+	if !validpkg {
+		opts.IncrBrokenPkgs()
+	}
+
+	return ans
+}
+
+func validateWorker(i int,
+	wg *sync.WaitGroup,
+	c <-chan pkg.Package,
+	opts *ValidateOpts) {
+
+	defer wg.Done()
+
+	for p := range c {
+
+		if opts.OnlyBuildtime {
+			// Check buildtime compiler/deps
+			err := validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+		} else if opts.OnlyRuntime {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+		} else {
+
+			// Check runtime installer/deps
+			err := validatePackage(p, "runtime", opts, opts.RuntimeReciper, opts.RuntimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+				continue
+			}
+
+			// Check buildtime compiler/deps
+			err = validatePackage(p, "buildtime", opts, opts.BuildtimeReciper, opts.BuildtimeCacheDeps)
+			if err != nil {
+				opts.AddError(err)
+			}
+
+		}
+
+	}
+}
+
+func initOpts(opts *ValidateOpts, onlyRuntime, onlyBuildtime, withSolver bool, treePaths []string) {
+	var err error
+
+	opts.OnlyBuildtime = onlyBuildtime
+	opts.OnlyRuntime = onlyRuntime
+	opts.WithSolver = withSolver
+	opts.RuntimeReciper = nil
+	opts.BuildtimeReciper = nil
+	opts.BrokenPkgs = 0
+	opts.BrokenDeps = 0
+
+	if onlyBuildtime {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+	} else if onlyRuntime {
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	} else {
+		opts.BuildtimeReciper = (tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.CompilerRecipe)
+		opts.RuntimeReciper = (tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false))).(*tree.InstallerRecipe)
+	}
+
+	opts.RuntimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+	opts.BuildtimeCacheDeps = pkg.NewInMemoryDatabase(false).(*pkg.InMemoryDatabase)
+
+	for _, treePath := range treePaths {
+		util.DefaultContext.Info(fmt.Sprintf("Loading :deciduous_tree: %s...", treePath))
+		if opts.BuildtimeReciper != nil {
+			err = opts.BuildtimeReciper.Load(treePath)
+			if err != nil {
+				util.DefaultContext.Fatal("Error on load tree ", err)
+			}
+		}
+		if opts.RuntimeReciper != nil {
+			err = opts.RuntimeReciper.Load(treePath)
+			if err != nil {
+				util.DefaultContext.Fatal("Error on load tree ", err)
+			}
+		}
+	}
+
+	opts.RegExcludes, err = helpers.CreateRegexArray(opts.Excludes)
+	if err != nil {
+		util.DefaultContext.Fatal(err.Error())
+	}
+	opts.RegMatches, err = helpers.CreateRegexArray(opts.Matches)
+	if err != nil {
+		util.DefaultContext.Fatal(err.Error())
+	}
+
+}
+
+func NewTreeValidateCommand() *cobra.Command {
+	var excludes []string
+	var matches []string
+	var treePaths []string
+	var opts ValidateOpts
+
+	var ans = &cobra.Command{
+		Use:   "validate [OPTIONS]",
+		Short: "Validate a tree or a list of packages",
+		Args:  cobra.OnlyValidArgs,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			if len(treePaths) < 1 {
+				util.DefaultContext.Fatal("Mandatory tree param missing.")
+			}
+			if onlyRuntime && onlyBuildtime {
+				util.DefaultContext.Fatal("Both --only-runtime and --only-buildtime options are not possibile.")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var reciper tree.Builder
+
+			concurrency := util.DefaultContext.Config.GetGeneral().Concurrency
+
+			withSolver, _ := cmd.Flags().GetBool("with-solver")
+			onlyRuntime, _ := cmd.Flags().GetBool("only-runtime")
+			onlyBuildtime, _ := cmd.Flags().GetBool("only-buildtime")
+
+			opts.Excludes = excludes
+			opts.Matches = matches
+			initOpts(&opts, onlyRuntime, onlyBuildtime, withSolver, treePaths)
+
+			// We need at least one valid reciper for get list of the packages.
+			if onlyBuildtime {
+				reciper = opts.BuildtimeReciper
+			} else {
+				reciper = opts.RuntimeReciper
+			}
+
+			all := make(chan pkg.Package)
+
+			var wg = new(sync.WaitGroup)
+
+			for i := 0; i < concurrency; i++ {
+				wg.Add(1)
+				go validateWorker(i, wg, all, &opts)
+			}
+			for _, p := range reciper.GetDatabase().World() {
+				all <- p
+			}
+			close(all)
+
+			// Wait separately and once done close the channel
+			go func() {
+				wg.Wait()
+			}()
+
+			stringerrs := []string{}
+			for _, e := range opts.Errors {
+				stringerrs = append(stringerrs, e.Error())
+			}
+			sort.Strings(stringerrs)
+			for _, e := range stringerrs {
+				fmt.Println(e)
+			}
+
+			// fmt.Println("Broken packages:", brokenPkgs, "(", brokenDeps, "deps ).")
+			if len(stringerrs) != 0 {
+				util.DefaultContext.Error(fmt.Sprintf("Found %d broken packages and %d broken deps.",
+					opts.BrokenPkgs, opts.BrokenDeps))
+				util.DefaultContext.Fatal("Errors: " + strconv.Itoa(len(stringerrs)))
+			} else {
+				util.DefaultContext.Info("All good! :white_check_mark:")
+				os.Exit(0)
+			}
+		},
+	}
+	path, err := os.Getwd()
+	if err != nil {
+		util.DefaultContext.Fatal(err)
+	}
+	ans.Flags().Bool("only-runtime", false, "Check only runtime dependencies.")
+	ans.Flags().Bool("only-buildtime", false, "Check only buildtime dependencies.")
+	ans.Flags().BoolP("with-solver", "s", false,
+		"Enable check of requires also with solver.")
+	ans.Flags().StringSliceVarP(&treePaths, "tree", "t", []string{path},
+		"Path of the tree to use.")
+	ans.Flags().StringSliceVarP(&excludes, "exclude", "e", []string{},
+		"Exclude matched packages from analysis. (Use string as regex).")
+	ans.Flags().StringSliceVarP(&matches, "matches", "m", []string{},
+		"Analyze only matched packages. (Use string as regex).")
+
+	return ans
+}

cmd/uninstall.go

@@ -0,0 +1,113 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	helpers "github.com/mudler/luet/cmd/helpers"
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var uninstallCmd = &cobra.Command{
+	Use:     "uninstall <pkg> <pkg2> ...",
+	Short:   "Uninstall a package or a list of packages",
+	Long:    `Uninstall packages`,
+	Aliases: []string{"rm", "un"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		toRemove := []pkg.Package{}
+		for _, a := range args {
+
+			pack, err := helpers.ParsePackageStr(a)
+			if err != nil {
+				util.DefaultContext.Fatal("Invalid package string ", a, ": ", err.Error())
+			}
+			toRemove = append(toRemove, pack)
+		}
+
+		force := viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		checkconflicts, _ := cmd.Flags().GetBool("conflictscheck")
+		fullClean, _ := cmd.Flags().GetBool("full-clean")
+		yes := viper.GetBool("yes")
+		keepProtected, _ := cmd.Flags().GetBool("keep-protected-files")
+
+		util.SetSystemConfig(util.DefaultContext)
+		util.SetSolverConfig(util.DefaultContext)
+
+		util.DefaultContext.Config.ConfigProtectSkip = !keepProtected
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			NoDeps:                      nodeps,
+			Force:                       force,
+			FullUninstall:               full,
+			FullCleanUninstall:          fullClean,
+			CheckConflicts:              checkconflicts,
+			Ask:                         !yes,
+			PreserveSystemEssentialData: true,
+			Context:                     util.DefaultContext,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+
+		if err := inst.Uninstall(system, toRemove...); err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+
+	uninstallCmd.Flags().String("system-dbpath", "", "System db path")
+	uninstallCmd.Flags().String("system-target", "", "System rootpath")
+	uninstallCmd.Flags().String("system-engine", "", "System DB engine")
+
+	uninstallCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	uninstallCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	uninstallCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	uninstallCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	uninstallCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	uninstallCmd.Flags().Bool("force", false, "Force uninstall")
+	uninstallCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	uninstallCmd.Flags().Bool("conflictscheck", true, "Check if the package marked for deletion is required by other packages")
+	uninstallCmd.Flags().Bool("full-clean", false, "(experimental) Uninstall packages and all the other deps/revdeps of it.")
+	uninstallCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	uninstallCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	uninstallCmd.Flags().BoolP("keep-protected-files", "k", false, "Keep package protected files around")
+
+	RootCmd.AddCommand(uninstallCmd)
+}

cmd/upgrade.go

@@ -0,0 +1,102 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var upgradeCmd = &cobra.Command{
+	Use:     "upgrade",
+	Short:   "Upgrades the system",
+	Aliases: []string{"u"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Long: `Upgrades packages in parallel`,
+	Run: func(cmd *cobra.Command, args []string) {
+
+		force := viper.GetBool("force")
+		nodeps, _ := cmd.Flags().GetBool("nodeps")
+		full, _ := cmd.Flags().GetBool("full")
+		universe, _ := cmd.Flags().GetBool("universe")
+		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
+		yes := viper.GetBool("yes")
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+
+		util.SetSystemConfig(util.DefaultContext)
+		opts := util.SetSolverConfig(util.DefaultContext)
+
+		util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+		util.DefaultContext.Debug("Solver", opts.CompactString())
+
+		// Load config protect configs
+		util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+			Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+			SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+			Force:                       force,
+			FullUninstall:               full,
+			NoDeps:                      nodeps,
+			SolverUpgrade:               universe,
+			RemoveUnavailableOnUpgrade:  clean,
+			UpgradeNewRevisions:         sync,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
+			DownloadOnly:                downloadOnly,
+			PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			Context:                     util.DefaultContext,
+		})
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		if err := inst.Upgrade(system); err != nil {
+			util.DefaultContext.Fatal("Error: " + err.Error())
+		}
+	},
+}
+
+func init() {
+	upgradeCmd.Flags().String("system-dbpath", "", "System db path")
+	upgradeCmd.Flags().String("system-target", "", "System rootpath")
+	upgradeCmd.Flags().String("system-engine", "", "System DB engine")
+
+	upgradeCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	upgradeCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	upgradeCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	upgradeCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	upgradeCmd.Flags().Bool("force", false, "Force upgrade by ignoring errors")
+	upgradeCmd.Flags().Bool("nodeps", false, "Don't consider package dependencies (harmful! overrides checkconflicts and full!)")
+	upgradeCmd.Flags().Bool("full", false, "Attempts to remove as much packages as possible which aren't required (slow)")
+	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
+	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
+	upgradeCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	upgradeCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	upgradeCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(upgradeCmd)
+}

cmd/util.go

@@ -0,0 +1,111 @@
+// Copyright © 2020-2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/go-units"
+
+	"github.com/mudler/luet/cmd/util"
+	"github.com/mudler/luet/pkg/helpers/docker"
+
+	"github.com/spf13/cobra"
+)
+
+func NewUnpackCommand() *cobra.Command {
+
+	c := &cobra.Command{
+		Use:   "unpack image path",
+		Short: "Unpack a docker image natively",
+		Long: `unpack doesn't need the docker daemon to run, and unpacks a docker image in the specified directory:
+		
+	luet util unpack golang:alpine /alpine
+`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+
+			if len(args) != 2 {
+				util.DefaultContext.Fatal("Expects an image and a path")
+			}
+
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+
+			image := args[0]
+			destination, err := filepath.Abs(args[1])
+			if err != nil {
+				util.DefaultContext.Error("Invalid path %s", destination)
+				os.Exit(1)
+			}
+
+			verify, _ := cmd.Flags().GetBool("verify")
+			user, _ := cmd.Flags().GetString("auth-username")
+			pass, _ := cmd.Flags().GetString("auth-password")
+			authType, _ := cmd.Flags().GetString("auth-type")
+			server, _ := cmd.Flags().GetString("auth-server-address")
+			identity, _ := cmd.Flags().GetString("auth-identity-token")
+			registryToken, _ := cmd.Flags().GetString("auth-registry-token")
+
+			temp, err := util.DefaultContext.Config.GetSystem().TempDir("contentstore")
+			if err != nil {
+				util.DefaultContext.Fatal("Cannot create a tempdir", err.Error())
+			}
+
+			util.DefaultContext.Info("Downloading", image, "to", destination)
+			auth := &types.AuthConfig{
+				Username:      user,
+				Password:      pass,
+				ServerAddress: server,
+				Auth:          authType,
+				IdentityToken: identity,
+				RegistryToken: registryToken,
+			}
+
+			info, err := docker.DownloadAndExtractDockerImage(temp, image, destination, auth, verify)
+			if err != nil {
+				util.DefaultContext.Error(err.Error())
+				os.Exit(1)
+			}
+			util.DefaultContext.Info(fmt.Sprintf("Pulled: %s %s", info.Target.Digest, info.Name))
+			util.DefaultContext.Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.Target.Size))))
+		},
+	}
+
+	c.Flags().String("auth-username", "", "Username to authenticate to registry/notary")
+	c.Flags().String("auth-password", "", "Password to authenticate to registry")
+	c.Flags().String("auth-type", "", "Auth type")
+	c.Flags().String("auth-server-address", "", "Authentication server address")
+	c.Flags().String("auth-identity-token", "", "Authentication identity token")
+	c.Flags().String("auth-registry-token", "", "Authentication registry token")
+	c.Flags().Bool("verify", false, "Verify signed images to notary before to pull")
+	return c
+}
+
+var utilGroup = &cobra.Command{
+	Use:   "util [command] [OPTIONS]",
+	Short: "General luet internal utilities exposed",
+}
+
+func init() {
+	RootCmd.AddCommand(utilGroup)
+
+	utilGroup.AddCommand(
+		NewUnpackCommand(),
+	)
+}

cmd/util/cli.go

@@ -0,0 +1,166 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/marcsauter/single"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/installer"
+)
+
+var DefaultContext = types.NewContext()
+
+var lockedCommands = []string{"install", "uninstall", "upgrade"}
+var bannerCommands = []string{"install", "build", "uninstall", "upgrade"}
+
+func BindSystemFlags(cmd *cobra.Command) {
+	viper.BindPFlag("system.database_path", cmd.Flags().Lookup("system-dbpath"))
+	viper.BindPFlag("system.rootfs", cmd.Flags().Lookup("system-target"))
+	viper.BindPFlag("system.database_engine", cmd.Flags().Lookup("system-engine"))
+}
+
+func BindSolverFlags(cmd *cobra.Command) {
+	viper.BindPFlag("solver.type", cmd.Flags().Lookup("solver-type"))
+	viper.BindPFlag("solver.discount", cmd.Flags().Lookup("solver-discount"))
+	viper.BindPFlag("solver.rate", cmd.Flags().Lookup("solver-rate"))
+	viper.BindPFlag("solver.max_attempts", cmd.Flags().Lookup("solver-attempts"))
+}
+
+func BindValuesFlags(cmd *cobra.Command) {
+	viper.BindPFlag("values", cmd.Flags().Lookup("values"))
+}
+
+func ValuesFlags() []string {
+	return viper.GetStringSlice("values")
+}
+
+func SetSystemConfig(ctx *types.Context) {
+	dbpath := viper.GetString("system.database_path")
+	rootfs := viper.GetString("system.rootfs")
+	engine := viper.GetString("system.database_engine")
+
+	ctx.Config.System.DatabaseEngine = engine
+	ctx.Config.System.DatabasePath = dbpath
+	ctx.Config.System.SetRootFS(rootfs)
+}
+
+func SetSolverConfig(ctx *types.Context) (c *types.LuetSolverOptions) {
+	stype := viper.GetString("solver.type")
+	discount := viper.GetFloat64("solver.discount")
+	rate := viper.GetFloat64("solver.rate")
+	attempts := viper.GetInt("solver.max_attempts")
+
+	ctx.Config.GetSolverOptions().Type = stype
+	ctx.Config.GetSolverOptions().LearnRate = float32(rate)
+	ctx.Config.GetSolverOptions().Discount = float32(discount)
+	ctx.Config.GetSolverOptions().MaxAttempts = attempts
+
+	return &types.LuetSolverOptions{
+		Type:        stype,
+		LearnRate:   float32(rate),
+		Discount:    float32(discount),
+		MaxAttempts: attempts,
+	}
+}
+
+func SetCliFinalizerEnvs(ctx *types.Context, finalizerEnvs []string) error {
+	if len(finalizerEnvs) > 0 {
+		for _, v := range finalizerEnvs {
+			idx := strings.Index(v, "=")
+			if idx < 0 {
+				return errors.New("Found invalid runtime finalizer environment: " + v)
+			}
+
+			ctx.Config.SetFinalizerEnv(v[0:idx], v[idx+1:])
+		}
+
+	}
+
+	return nil
+}
+
+// TemplateFolders returns the default folders which holds shared template between packages in a given tree path
+func TemplateFolders(ctx *types.Context, fromRepo bool, treePaths []string) []string {
+	templateFolders := []string{}
+	for _, t := range treePaths {
+		templateFolders = append(templateFolders, filepath.Join(t, "templates"))
+	}
+	if fromRepo {
+		for _, s := range installer.SystemRepositories(ctx.Config.SystemRepositories) {
+			templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates"))
+		}
+	}
+	return templateFolders
+}
+
+func IntroScreen() {
+	luetLogo, _ := pterm.DefaultBigText.WithLetters(
+		pterm.NewLettersFromStringWithStyle("LU", pterm.NewStyle(pterm.FgLightMagenta)),
+		pterm.NewLettersFromStringWithStyle("ET", pterm.NewStyle(pterm.FgLightBlue))).
+		Srender()
+
+	pterm.DefaultCenter.Print(luetLogo)
+
+	pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint("Luet - 0-deps container-based package manager"))
+}
+
+func HandleLock(c *types.Context) {
+	if os.Getenv("LUET_NOLOCK") != "true" {
+		if len(os.Args) > 1 {
+			for _, lockedCmd := range lockedCommands {
+				if os.Args[1] == lockedCmd {
+					s := single.New("luet")
+					if err := s.CheckLock(); err != nil && err == single.ErrAlreadyRunning {
+						c.Fatal("another instance of the app is already running, exiting")
+					} else if err != nil {
+						// Another error occurred, might be worth handling it as well
+						c.Fatal("failed to acquire exclusive app lock:", err.Error())
+					}
+					defer s.TryUnlock()
+					break
+				}
+			}
+		}
+	}
+}
+
+func DisplayVersionBanner(c *types.Context, banner func(), version func() string, license []string) {
+	display := false
+	if len(os.Args) > 1 {
+		for _, c := range bannerCommands {
+			if os.Args[1] == c {
+				display = true
+			}
+		}
+	}
+	if display {
+		banner()
+		pterm.DefaultCenter.Print(version())
+		for _, l := range license {
+			pterm.DefaultCenter.Print(l)
+
+		}
+	}
+}

cmd/util/config.go

@@ -0,0 +1,207 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	extensions "github.com/mudler/cobra-extensions"
+	"github.com/mudler/luet/pkg/api/core/types"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	LuetEnvPrefix = "LUET"
+)
+
+var cfgFile string
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	setDefaults(viper.GetViper())
+	// Luet support these priorities on read configuration file:
+	// - command line option (if available)
+	// - $PWD/.luet.yaml
+	// - $HOME/.luet.yaml
+	// - /etc/luet/luet.yaml
+	//
+	// Note: currently a single viper instance support only one config name.
+
+	viper.SetEnvPrefix(LuetEnvPrefix)
+	viper.SetConfigType("yaml")
+
+	if cfgFile != "" { // enable ability to specify config file via flag
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Retrieve pwd directory
+		pwdDir, err := os.Getwd()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		homeDir := helpers.GetHomeDir()
+
+		if fileHelper.Exists(filepath.Join(pwdDir, ".luet.yaml")) || (homeDir != "" && fileHelper.Exists(filepath.Join(homeDir, ".luet.yaml"))) {
+			viper.AddConfigPath(".")
+			if homeDir != "" {
+				viper.AddConfigPath(homeDir)
+			}
+			viper.SetConfigName(".luet")
+		} else {
+			viper.SetConfigName("luet")
+			viper.AddConfigPath("/etc/luet")
+		}
+	}
+
+	viper.AutomaticEnv() // read in environment variables that match
+
+	// Create EnvKey Replacer for handle complex structure
+	replacer := strings.NewReplacer(".", "__")
+	viper.SetEnvKeyReplacer(replacer)
+	viper.SetTypeByDefaultValue(true)
+	// If a config file is found, read it in.
+	viper.ReadInConfig()
+
+}
+
+// InitContext inits the context by parsing the configurations from viper
+// this is meant to be run before each command to be able to parse any override from
+// the CLI/ENV
+func InitContext(ctx *types.Context) (err error) {
+
+	err = viper.Unmarshal(&ctx.Config)
+	if err != nil {
+		return
+	}
+
+	// Inits the context with the configurations loaded
+	// It reads system repositories, sets logging, and all the
+	// context which is required to perform luet actions
+	err = ctx.Init()
+	if err != nil {
+		return
+	}
+
+	// no_spinner is not mapped in our configs
+	ctx.NoSpinner = viper.GetBool("no_spinner")
+	return
+}
+
+func setDefaults(viper *viper.Viper) {
+	viper.SetDefault("logging.level", "info")
+	viper.SetDefault("logging.enable_logfile", false)
+	viper.SetDefault("logging.path", "/var/log/luet.log")
+	viper.SetDefault("logging.json_format", false)
+	viper.SetDefault("logging.enable_emoji", true)
+	viper.SetDefault("logging.color", true)
+
+	viper.SetDefault("general.concurrency", runtime.NumCPU())
+	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.show_build_output", false)
+	viper.SetDefault("general.fatal_warnings", false)
+	viper.SetDefault("general.http_timeout", 360)
+
+	u, err := user.Current()
+	// os/user doesn't work in from scratch environments
+	if err != nil || (u != nil && u.Uid == "0") {
+		viper.SetDefault("general.same_owner", true)
+	} else {
+		viper.SetDefault("general.same_owner", false)
+	}
+
+	viper.SetDefault("system.database_engine", "boltdb")
+	viper.SetDefault("system.database_path", "/var/cache/luet")
+	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
+	viper.SetDefault("system.pkgs_cache_path", "packages")
+
+	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
+	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
+	// TODO: Set default to false when we are ready for migration.
+	viper.SetDefault("config_from_host", true)
+	viper.SetDefault("cache_repositories", []string{})
+	viper.SetDefault("system_repositories", []string{})
+	viper.SetDefault("finalizer_envs", make(map[string]string))
+
+	viper.SetDefault("solver.type", "")
+	viper.SetDefault("solver.rate", 0.7)
+	viper.SetDefault("solver.discount", 1.0)
+	viper.SetDefault("solver.max_attempts", 9000)
+}
+
+// InitViper inits a new viper
+// this is meant to be run just once at beginning to setup the root command
+func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
+	cobra.OnInitialize(initConfig)
+	pflags := RootCmd.PersistentFlags()
+	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
+	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.Bool("fatal", false, "Enables Warnings to exit")
+	pflags.Bool("enable-logfile", false, "Enable log to file")
+	pflags.Bool("no-spinner", false, "Disable spinner.")
+	pflags.Bool("color", ctx.Config.GetLogging().Color, "Enable/Disable color.")
+	pflags.Bool("emoji", ctx.Config.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", ctx.Config.ConfigProtectSkip,
+		"Disable config protect analysis.")
+	pflags.StringP("logfile", "l", ctx.Config.GetLogging().Path,
+		"Logfile path. Empty value disable log to file.")
+	pflags.StringSlice("plugin", []string{}, "A list of runtime plugins to load")
+
+	// os/user doesn't work in from scratch environments.
+	// Check if i can retrieve user informations.
+	_, err := user.Current()
+	if err != nil {
+		ctx.Warning("failed to retrieve user identity:", err.Error())
+	}
+	pflags.Bool("same-owner", ctx.Config.GetGeneral().SameOwner, "Maintain same owner on uncompress.")
+	pflags.Int("concurrency", runtime.NumCPU(), "Concurrency")
+	pflags.Int("http-timeout", ctx.Config.General.HTTPTimeout, "Default timeout for http(s) requests")
+
+	viper.BindPFlag("logging.color", pflags.Lookup("color"))
+	viper.BindPFlag("logging.enable_emoji", pflags.Lookup("emoji"))
+	viper.BindPFlag("logging.enable_logfile", pflags.Lookup("enable-logfile"))
+	viper.BindPFlag("logging.path", pflags.Lookup("logfile"))
+
+	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
+	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
+	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
+	viper.BindPFlag("plugin", pflags.Lookup("plugin"))
+	viper.BindPFlag("general.http_timeout", pflags.Lookup("http-timeout"))
+
+	// Currently I maintain this only from cli.
+	viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
+
+	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
+	// we also accept extensions in the relative path where luet is being started, "extensions/"
+	exts := extensions.Discover("luet", "extensions")
+	for _, ex := range exts {
+		cobraCmd := ex.CobraCommand()
+		RootCmd.AddCommand(cobraCmd)
+	}
+}

cmd/util/search.go

@@ -0,0 +1,42 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package util
+
+import "github.com/pterm/pterm"
+
+type TableWriter struct {
+	td pterm.TableData
+}
+
+func (l *TableWriter) AppendRow(item []string) {
+	l.td = append(l.td, item)
+}
+
+func (l *TableWriter) Render() {
+	pterm.DefaultTable.WithHasHeader().WithData(l.td).Render()
+}
+
+type ListWriter struct {
+	bb []pterm.BulletListItem
+}
+
+func (l *ListWriter) AppendItem(item pterm.BulletListItem) {
+	l.bb = append(l.bb, item)
+}
+
+func (l *ListWriter) Render() {
+	pterm.DefaultBulletList.WithItems(l.bb).Render()
+}

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/get_luet_root.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+if [ $(id -u) -ne 0 ]
+  then echo "Please run the installer with sudo/as root"
+  exit
+fi
+
+set -ex
+export LUET_NOLOCK=true
+
+LUET_VERSION=$(curl -s https://api.github.com/repos/mudler/luet/releases/latest | grep tag_name | awk '{ print $2 }' | sed -e 's/\"//g' -e 's/,//g' || echo "0.9.24" )
+LUET_ROOTFS=${LUET_ROOTFS:-/}
+LUET_DATABASE_PATH=${LUET_DATABASE_PATH:-/var/luet/db}
+LUET_DATABASE_ENGINE=${LUET_DATABASE_ENGINE:-boltdb}
+LUET_CONFIG_PROTECT=${LUET_CONFIG_PROTECT:-1}
+
+curl -L https://github.com/mudler/luet/releases/download/${LUET_VERSION}/luet-${LUET_VERSION}-linux-amd64 --output luet
+chmod +x luet
+
+mkdir -p /etc/luet/repos.conf.d || true
+mkdir -p $LUET_DATABASE_PATH || true
+mkdir -p /var/tmp/luet || true
+
+if [ "${LUET_CONFIG_PROTECT}" = "1" ] ; then
+  mkdir -p /etc/luet/config.protect.d || true
+  curl -L https://raw.githubusercontent.com/mudler/luet/master/contrib/config/config.protect.d/01_etc.yml.example --output /etc/luet/config.protect.d/01_etc.yml
+fi
+curl -L https://raw.githubusercontent.com/mocaccinoOS/repository-index/master/packages/mocaccino-repository-index.yml --output /etc/luet/repos.conf.d/mocaccino-repository-index.yml
+
+cat > /etc/luet/luet.yaml <<EOF
+general:
+  debug: false
+system:
+  rootfs: ${LUET_ROOTFS}
+  database_path: "${LUET_DATABASE_PATH}"
+  database_engine: "${LUET_DATABASE_ENGINE}"
+  tmpdir_base: "/var/tmp/luet"
+EOF
+
+./luet install -y repository/luet repository/mocaccino-repository-index
+./luet install -y system/luet system/luet-extensions
+
+rm -rf luet
+

contrib/config/luet.yaml

@@ -0,0 +1,171 @@
+# Luet Configuration File
+#
+# ---------------------------------------------
+# Logging configuration section:
+# ---------------------------------------------
+# logging:
+#   # Enable loggging to file (if path is not empty)
+#   enable_logfile: false
+#
+#   Leave empty to skip logging to file.
+#   path: "/var/log/luet.log"
+#
+#   Set logging level: error|warning|info|debug
+#   level: "info"
+#
+#   Enable JSON log format instead of console mode.
+#   json_format: false.
+#
+#   Disable/Enable color
+#   color: true
+#
+#   Enable/Disable emoji
+#   enable_emoji: true
+#
+# ---------------------------------------------
+# General configuration section:
+# ---------------------------------------------
+# general:
+#   Define max concurrency processes. Default is based of arch: runtime.NumCPU()
+#   concurrency: 1
+#
+#   Enable Debug. If debug is active spinner is disabled.
+#   debug: false
+#
+#   Show output of build execution (docker, img, etc.)
+#   show_build_output: false
+#
+#   Define spinner ms
+#   spinner_ms: 200
+#
+#   Define spinner charset. See https://github.com/briandowns/spinner
+#   spinner_charset: 22
+#
+#   Enable warnings to exit
+#   fatal_warnings: false
+#
+#   Try extracting tree/packages with the same ownership as exists in the archive (default for superuser).
+#   same_owner: false
+#
+# ---------------------------------------------
+# System configuration section:
+# ---------------------------------------------
+# system:
+#
+#   Rootfs path of the luet system. Default is /.
+#   A specific path could be used for test installation to
+#   a chroot environment.
+#   rootfs: "/"
+#
+#   Choice database engine used for luet database.
+#   Supported values: boltdb|memory
+#   database_engine: boltdb
+#
+#   Database path directory where store luet database.
+#   The path is append to rootfs option path.
+#   database_path: "/var/cache/luet"
+#
+#   Define the tmpdir base directory where luet store temporary files.
+#   Default $TMPDIR/tmpluet
+#   tmpdir_base: "/tmp/tmpluet"
+#
+#
+# ---------------------------------------------
+# Repositories configurations directories.
+# ---------------------------------------------
+# Define the list of directories where luet
+# try for files with .yml extension that define
+# luet repository.
+# repos_confdir:
+#   - /etc/luet/repos.conf.d
+#
+#
+# ------------------------------------------------
+# Config protect configuration files directories.
+# -----------------------------------------------
+# Define the list of directories where load
+# configuration files with the list of config
+# protect paths.
+# config_protect_confdir:
+#   - /etc/luet/config.protect.d
+#
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
+# The paths used for load repositories and config
+# protects are based on host rootfs.
+# If set to false rootfs path is used as prefix.
+# config_from_host: true
+#
+#
+# ------------------------------------------------
+# Finalizer Environment Variables
+# -----------------------------------------------
+# finalizer_envs:
+#  - key: "BUILD_ISO"
+#    value: "1"
+#
+# System repositories
+# ---------------------------------------------
+# In alternative to define repositories files
+# through repos_confdir option is possible
+# define directly the list of the repositories.
+#
+# repositories:
+#
+#     Name of the repository. It's better that this name is unique. Mandatory.
+#   - name: "repo1"
+#
+#     A user-friendly description of the repository
+#     description: "My luet repo"
+#
+#     Type of the repository. Supported types are: dir|http. Mandatory.
+#     type: "dir"
+#
+#     Define the priority of the repository on research packages. Default is 9999.
+#     priority: 9999
+#
+#     Enable/Disable of the repository.
+#     enable: false
+#
+#     Cached repository. If true a local cache of the remote repository tree is maintained
+#     locally in the $tree_path else it is used a temporary directory that is removed when
+#     installation of a package is completed. A cached repository reduce time on search/install
+#     packages. By default caching is disable.
+#     cached: false
+#
+#     Path where store tree of the specifications. Default path is $database_path/repos/$repo_name/treefs
+#     tree_path: "/var/cache/luet/repos/local/treefs"
+#
+#     Path where store repository metadata. Default path is $database_path/repos/$repo_name/meta
+#     meta_path: "/var/cache/luet/repos/local/meta"
+#
+#     Define the list of the URL where retrieve tree and packages.
+#     urls:
+#        - https://mydomain.local/luet/repo1
+#
+#     auth:
+#        Define Basic authentication header
+#        basic: "mybasicauth"
+#        Define token authentication header
+#        token: "mytoken"
+# ---------------------------------------------
+# Solver parameter configuration:
+# ---------------------------------------------
+# solver:
+#
+#   Solver strategy to solve possible conflicts during depedency
+#   solving. Defaults to empty (none). Available: qlearning
+#   type: ""
+#
+#   Solver agent learning rate. 0.1 to 1.0
+#   rate: 0.7
+#
+#   Learning discount factor.
+#   discount: 1.0
+#
+#   Number of overall attempts that the solver has available before bailing out.
+#   max_attempts: 9000
+#

go.mod

@@ -1,36 +1,85 @@
 module github.com/mudler/luet
 
-go 1.12
+go 1.16
 
 require (
-	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/MottainaiCI/simplestreams-builder v0.0.0-20190710131531-efb382161f56 // indirect
-	github.com/Sabayon/pkgs-checker v0.4.1
+	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.1.1 // indirect
+	github.com/Microsoft/go-winio v0.5.0 // indirect
+	github.com/Sabayon/pkgs-checker v0.8.4
+	github.com/apex/log v1.9.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
-	github.com/briandowns/spinner v1.7.0
-	github.com/crillab/gophersat v1.1.7
-	github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23
+	github.com/cavaliercoder/grab v1.0.1-0.20201108051000-98a5bfe305ec
+	github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d // indirect
+	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
+	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
+	github.com/crillab/gophersat v1.3.2-0.20210701121804-72b19f5b6b38
+	github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
+	github.com/docker/go-units v0.4.0
+	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.0
-	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
-	github.com/hashicorp/go-version v1.2.0
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-containerregistry v0.2.1
+	github.com/google/renameio v1.0.0
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gookit/color v1.5.0 // indirect
+	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-version v1.3.0
+	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/imdario/mergo v0.3.12
 	github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3
+	github.com/klauspost/compress v1.12.2
+	github.com/klauspost/pgzip v1.2.1
+	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/kyokomi/emoji v2.1.0+incompatible
-	github.com/logrusorgru/aurora v0.0.0-20190417123914-21d75270181e
-	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/mudler/docker-companion v0.4.6-0.20191110154655-b8b364100616
-	github.com/onsi/ginkgo v1.10.1
-	github.com/onsi/gomega v1.7.0
-	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/marcsauter/single v0.0.0-20181104081128-f8bf46f26ec0
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.1
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
+	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
+	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/gomega v1.16.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/runc v1.0.0-rc9.0.20200221051241-688cf6d43cc4 // indirect
+	github.com/otiai10/copy v1.2.1-0.20200916181228-26f84a0b1578
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
-	github.com/pkg/errors v0.8.1
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.5.0
-	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	go.etcd.io/bbolt v1.3.3
-	golang.org/x/crypto v0.0.0-20191028145041-f83a4685e152 // indirect
-	golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 // indirect
-	gopkg.in/yaml.v2 v2.2.5
-	mvdan.cc/sh v2.6.4+incompatible
+	github.com/pkg/errors v0.9.1
+	github.com/pterm/pterm v0.12.32-0.20211002183613-ada9ef6790c3
+	github.com/rancher-sandbox/gofilecache v0.0.0-20210330135715-becdeff5df15
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.2.1
+	github.com/spf13/viper v1.8.1
+	github.com/theupdateframework/notary v0.7.0
+	go.etcd.io/bbolt v1.3.5
+	go.uber.org/multierr v1.6.0
+	go.uber.org/zap v1.17.0
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
+	golang.org/x/mod v0.4.2
+	golang.org/x/oauth2 v0.0.0-20210810183815-faf39c7919d5 // indirect
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/genproto v0.0.0-20210811021853-ddbe55d93216 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0
+	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
+
 )
+
+replace github.com/docker/docker => github.com/Luet-lab/moby v17.12.0-ce-rc1.0.20200605210607-749178b8f80d+incompatible

pkg/api/client/client_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestClient(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Client API Suite")
+}

pkg/api/client/search.go

@@ -0,0 +1,131 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/api/client/utils"
+)
+
+func TreePackages(treedir string) (searchResult SearchResult, err error) {
+	var res []byte
+	res, err = utils.RunSHOUT("tree", fmt.Sprintf("luet tree pkglist --tree %s --output json", treedir))
+	if err != nil {
+		fmt.Println(string(res))
+		return
+	}
+	json.Unmarshal(res, &searchResult)
+	return
+}
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+type SearchResult struct {
+	Packages []Package
+}
+
+type Package struct {
+	Name, Category, Version, Path string
+}
+
+func (p Package) String() string {
+	return fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version)
+}
+
+func (p Package) Image(repository string) string {
+	return fmt.Sprintf("%s:%s-%s-%s", repository, p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageTag() string {
+	// ${name}-${category}-${version//+/-}
+	return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, strings.ReplaceAll(p.Version, "+", "-"))
+}
+
+func (p Package) ImageMetadata(repository string) string {
+	return fmt.Sprintf("%s.metadata.yaml", p.Image(repository))
+}
+
+func (p Package) ImageAvailable(repository string) bool {
+	return imageAvailable(p.Image(repository))
+}
+
+func (p Package) Equal(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category && p.Version == pp.Version {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualS(s string) bool {
+	if s == fmt.Sprintf("%s/%s", p.Category, p.Name) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualSV(s string) bool {
+	if s == fmt.Sprintf("%s/%s@%s", p.Category, p.Name, p.Version) {
+		return true
+	}
+	return false
+}
+
+func (p Package) EqualNoV(pp Package) bool {
+	if p.Name == pp.Name && p.Category == pp.Category {
+		return true
+	}
+	return false
+}
+
+func (s SearchResult) FilterByCategory(cat string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if r.Category == cat {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+func (s SearchResult) FilterByName(name string) SearchResult {
+	new := SearchResult{Packages: []Package{}}
+
+	for _, r := range s.Packages {
+		if !strings.Contains(r.Name, name) {
+			new.Packages = append(new.Packages, r)
+		}
+	}
+	return new
+}
+
+type Packages []Package
+
+func (p Packages) Exist(pp Package) bool {
+	for _, pi := range p {
+		if pp.Equal(pi) {
+			return true
+		}
+	}
+	return false
+}

pkg/api/client/search_test.go

@@ -0,0 +1,47 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package client_test
+
+import (
+	. "github.com/mudler/luet/pkg/api/client"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Client CLI API", func() {
+	Context("Reads a package tree from the luet CLI", func() {
+		It("Correctly detect packages", func() {
+			t, err := TreePackages("../../../tests/fixtures/alpine")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t).ToNot(BeNil())
+			Expect(len(t.Packages)).To(Equal(1))
+			Expect(t.Packages[0].Name).To(Equal("alpine"))
+			Expect(t.Packages[0].Category).To(Equal("seed"))
+			Expect(t.Packages[0].Version).To(Equal("1.0"))
+			Expect(t.Packages[0].ImageAvailable("foo")).To(BeFalse())
+			Expect(t.Packages[0].Equal(t.Packages[0])).To(BeTrue())
+			Expect(t.Packages[0].Equal(Package{})).To(BeFalse())
+			Expect(t.Packages[0].EqualNoV(Package{Name: "alpine", Category: "seed"})).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpine")).To(BeTrue())
+			Expect(t.Packages[0].EqualS("seed/alpinev")).To(BeFalse())
+			Expect(t.Packages[0].EqualSV("seed/alpine@1.0")).To(BeTrue())
+			Expect(t.Packages[0].Image("foo")).To(Equal("foo:alpine-seed-1.0"))
+			Expect(Packages(t.Packages).Exist(t.Packages[0])).To(BeTrue())
+			Expect(Packages(t.Packages).Exist(Package{})).To(BeFalse())
+
+		})
+	})
+})

pkg/api/client/utils/sh.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func RunSHOUT(stepName, bashFragment string) ([]byte, error) {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+
+	cmd.Env = os.Environ()
+	//	log.Printf("Running in background: %v", stepName)
+
+	return cmd.CombinedOutput()
+}
+
+func RunSH(stepName, bashFragment string) error {
+	cmd := exec.Command("sh", "-s")
+	cmd.Stdin = strings.NewReader(bashWrap(bashFragment))
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+	log.Printf("Running: %v (%v)", stepName, bashFragment)
+
+	return cmd.Run()
+}
+
+func bashWrap(cmd string) string {
+	return `
+set -o errexit
+set -o nounset
+` + cmd + `
+`
+}

pkg/api/core/config/config_protect.go

@@ -0,0 +1,116 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+type ConfigProtectConfFile struct {
+	Filename string
+
+	Name        string   `mapstructure:"name" yaml:"name" json:"name"`
+	Directories []string `mapstructure:"dirs" yaml:"dirs" json:"dirs"`
+}
+
+func NewConfigProtectConfFile(filename string) *ConfigProtectConfFile {
+	return &ConfigProtectConfFile{
+		Filename:    filename,
+		Name:        "",
+		Directories: []string{},
+	}
+}
+
+func (c *ConfigProtectConfFile) String() string {
+	return fmt.Sprintf("[%s] filename: %s, dirs: %s", c.Name, c.Filename,
+		c.Directories)
+}
+
+type ConfigProtect struct {
+	AnnotationDir string
+	MapProtected  map[string]bool
+}
+
+func NewConfigProtect(annotationDir string) *ConfigProtect {
+	if len(annotationDir) > 0 && annotationDir[0:1] != "/" {
+		annotationDir = "/" + annotationDir
+	}
+	return &ConfigProtect{
+		AnnotationDir: annotationDir,
+		MapProtected:  make(map[string]bool, 0),
+	}
+}
+
+func (c *ConfigProtect) AddAnnotationDir(d string) {
+	c.AnnotationDir = d
+}
+
+func (c *ConfigProtect) GetAnnotationDir() string {
+	return c.AnnotationDir
+}
+
+func (c *ConfigProtect) Map(files []string, protected []ConfigProtectConfFile) {
+
+	for _, file := range files {
+
+		if file[0:1] != "/" {
+			file = "/" + file
+		}
+
+		if len(protected) > 0 {
+			for _, conf := range protected {
+				for _, dir := range conf.Directories {
+					// Note file is without / at begin (on unpack)
+					if strings.HasPrefix(file, filepath.Clean(dir)) {
+						// docker archive modifier works with path without / at begin.
+						c.MapProtected[file] = true
+						goto nextFile
+					}
+				}
+			}
+		}
+
+		if c.AnnotationDir != "" && strings.HasPrefix(file, filepath.Clean(c.AnnotationDir)) {
+			c.MapProtected[file] = true
+		}
+	nextFile:
+	}
+
+}
+
+func (c *ConfigProtect) Protected(file string) bool {
+	if file[0:1] != "/" {
+		file = "/" + file
+	}
+	_, ans := c.MapProtected[file]
+	return ans
+}
+
+func (c *ConfigProtect) GetProtectFiles(withSlash bool) []string {
+	ans := []string{}
+
+	for key, _ := range c.MapProtected {
+		if withSlash {
+			ans = append(ans, key)
+		} else {
+			ans = append(ans, key[1:])
+		}
+	}
+	return ans
+}

pkg/api/core/config/config_protect_test.go

@@ -0,0 +1,121 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	config "github.com/mudler/luet/pkg/api/core/config"
+	"github.com/mudler/luet/pkg/api/core/types"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Test config protect", func() {
+
+		It("Protect1", func() {
+			ctx := types.NewContext()
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("/etc")
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+		It("Protect2", func() {
+			ctx := types.NewContext()
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("")
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeFalse())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{},
+			))
+		})
+
+		It("Protect3: Annotation dir without initial slash", func() {
+			ctx := types.NewContext()
+
+			files := []string{
+				"etc/foo/my.conf",
+				"usr/bin/foo",
+				"usr/share/doc/foo.md",
+			}
+
+			cp := config.NewConfigProtect("etc")
+			cp.Map(files, ctx.Config.ConfigProtectConfFiles)
+
+			Expect(cp.Protected("etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("/etc/foo/my.conf")).To(BeTrue())
+			Expect(cp.Protected("usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/bin/foo")).To(BeFalse())
+			Expect(cp.Protected("/usr/share/doc/foo.md")).To(BeFalse())
+
+			Expect(cp.GetProtectFiles(false)).To(Equal(
+				[]string{
+					"etc/foo/my.conf",
+				},
+			))
+
+			Expect(cp.GetProtectFiles(true)).To(Equal(
+				[]string{
+					"/etc/foo/my.conf",
+				},
+			))
+		})
+
+	})
+
+})

pkg/api/core/config/config_suite_test.go

@@ -0,0 +1,29 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Config Suite")
+}

pkg/api/core/types/artifact/artifact.go

@@ -0,0 +1,806 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	zstd "github.com/klauspost/compress/zstd"
+	gzip "github.com/klauspost/pgzip"
+
+	//"strconv"
+	"strings"
+	"sync"
+
+	config "github.com/mudler/luet/pkg/api/core/config"
+	types "github.com/mudler/luet/pkg/api/core/types"
+	bus "github.com/mudler/luet/pkg/bus"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	"github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
+// which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
+// In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
+type PackageArtifact struct {
+	Path string `json:"path"`
+
+	Dependencies      []*PackageArtifact                `json:"dependencies"`
+	CompileSpec       *compilerspec.LuetCompilationSpec `json:"compilationspec"`
+	Checksums         Checksums                         `json:"checksums"`
+	SourceAssertion   solver.PackagesAssertions         `json:"-"`
+	CompressionType   compression.Implementation        `json:"compressiontype"`
+	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
+	Runtime           *pkg.DefaultPackage               `json:"runtime,omitempty"`
+}
+
+func (p *PackageArtifact) ShallowCopy() *PackageArtifact {
+	copy := *p
+	return &copy
+}
+
+func NewPackageArtifact(path string) *PackageArtifact {
+	return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
+}
+
+func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
+	p := &PackageArtifact{Checksums: Checksums{}}
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return p, err
+	}
+
+	return p, err
+}
+
+func (a *PackageArtifact) Hash() error {
+	return a.Checksums.Generate(a)
+}
+
+func (a *PackageArtifact) Verify() error {
+	sum := Checksums{}
+	if err := sum.Generate(a); err != nil {
+		return err
+	}
+
+	if err := sum.Compare(a.Checksums); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *PackageArtifact) WriteYAML(dst string) error {
+	// First compute checksum of artifact. When we write the yaml we want to write up-to-date informations.
+	err := a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for artifact")
+	}
+
+	// Update runtime package information
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		runtime, err := a.CompileSpec.Package.GetRuntimePackage()
+		if err != nil {
+			return errors.Wrapf(err, "getting runtime package for '%s'", a.CompileSpec.Package.HumanReadableString())
+		}
+		a.Runtime = runtime
+	}
+
+	data, err := yaml.Marshal(a)
+	if err != nil {
+		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
+	}
+
+	bus.Manager.Publish(bus.EventPackagePreBuildArtifact, a)
+
+	mangle, err := NewPackageArtifactFromYaml(data)
+	if err != nil {
+		return errors.Wrap(err, "Generated invalid artifact")
+	}
+	//p := a.CompileSpec.GetPackage().GetPath()
+
+	mangle.CompileSpec.GetPackage().SetPath("")
+	for _, ass := range mangle.CompileSpec.GetSourceAssertion() {
+		ass.Package.SetPath("")
+	}
+
+	data, err = yaml.Marshal(mangle)
+	if err != nil {
+		return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
+	}
+
+	err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetMetadataFilePath()), data, os.ModePerm)
+	if err != nil {
+		return errors.Wrap(err, "While writing PackageArtifact YAML")
+	}
+	//a.CompileSpec.GetPackage().SetPath(p)
+	bus.Manager.Publish(bus.EventPackagePostBuildArtifact, a)
+
+	return nil
+}
+
+func (a *PackageArtifact) GetFileName() string {
+	return path.Base(a.Path)
+}
+
+func (a *PackageArtifact) genDockerfile() string {
+	return `
+FROM scratch
+COPY . /`
+}
+
+// CreateArtifactForFile creates a new artifact from the given file
+func CreateArtifactForFile(ctx *types.Context, s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
+	if _, err := os.Stat(s); os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "artifact path doesn't exist")
+	}
+	fileName := path.Base(s)
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	defer os.RemoveAll(archive) // clean up
+	dst := filepath.Join(archive, fileName)
+	if err := fileHelper.CopyFile(s, dst); err != nil {
+		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
+	}
+
+	artifact, err := ctx.Config.GetSystem().TempDir("artifact")
+	if err != nil {
+		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
+	}
+	a := &PackageArtifact{Path: filepath.Join(artifact, fileName)}
+
+	for _, o := range opts {
+		o(a)
+	}
+
+	return a, a.Compress(archive, 1)
+}
+
+type ImageBuilder interface {
+	BuildImage(backend.Options) error
+}
+
+// GenerateFinalImage takes an artifact and builds a Docker image with its content
+func (a *PackageArtifact) GenerateFinalImage(ctx *types.Context, imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
+	builderOpts := backend.Options{}
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+	defer os.RemoveAll(archive) // clean up
+
+	uncompressedFiles := filepath.Join(archive, "files")
+	dockerFile := filepath.Join(archive, "Dockerfile")
+
+	if err := os.MkdirAll(uncompressedFiles, os.ModePerm); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
+	}
+
+	if err := a.Unpack(ctx, uncompressedFiles, keepPerms); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
+	}
+
+	empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
+	if err != nil {
+		return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
+	}
+
+	// See https://github.com/moby/moby/issues/38039.
+	// We can't generate FROM scratch empty images. Docker will refuse to export them
+	// workaround: Inject a .virtual empty file
+	if empty {
+		fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
+	}
+
+	data := a.genDockerfile()
+	if err := ioutil.WriteFile(dockerFile, []byte(data), 0644); err != nil {
+		return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
+	}
+
+	builderOpts = backend.Options{
+		ImageName:      imageName,
+		SourcePath:     archive,
+		DockerFileName: dockerFile,
+		Context:        uncompressedFiles,
+	}
+	return builderOpts, b.BuildImage(builderOpts)
+}
+
+// Compress is responsible to archive and compress to the artifact Path.
+// It accepts a source path, which is the content to be archived/compressed
+// and a concurrency parameter.
+func (a *PackageArtifact) Compress(src string, concurrency int) error {
+	switch a.CompressionType {
+
+	case compression.Zstandard:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		zstdFile := a.getCompressedName()
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(zstdFile)
+		if err != nil {
+			return err
+		}
+
+		enc, err := zstd.NewWriter(dst)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(enc, bufferedReader)
+		if err != nil {
+			enc.Close()
+			return err
+		}
+		if err := enc.Close(); err != nil {
+			return err
+		}
+
+		os.RemoveAll(a.Path) // Remove original
+		//	Debug("Removed artifact", a.Path)
+
+		a.Path = zstdFile
+		return nil
+	case compression.GZip:
+		err := helpers.Tar(src, a.Path)
+		if err != nil {
+			return err
+		}
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return err
+		}
+		defer original.Close()
+
+		gzipfile := a.getCompressedName()
+		bufferedReader := bufio.NewReader(original)
+
+		// Open a file for writing.
+		dst, err := os.Create(gzipfile)
+		if err != nil {
+			return err
+		}
+		// Create gzip writer.
+		w := gzip.NewWriter(dst)
+		w.SetConcurrency(1<<20, concurrency)
+		defer w.Close()
+		defer dst.Close()
+		_, err = io.Copy(w, bufferedReader)
+		if err != nil {
+			return err
+		}
+		w.Close()
+		os.RemoveAll(a.Path) // Remove original
+		//	Debug("Removed artifact", a.Path)
+		//	a.CompressedPath = gzipfile
+		a.Path = gzipfile
+		return nil
+		//a.Path = gzipfile
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.Tar(src, a.getCompressedName())
+	}
+}
+
+func (a *PackageArtifact) getCompressedName() string {
+	switch a.CompressionType {
+	case compression.Zstandard:
+		return a.Path + ".zst"
+
+	case compression.GZip:
+		return a.Path + ".gz"
+	}
+	return a.Path
+}
+
+// GetUncompressedName returns the artifact path without the extension suffix
+func (a *PackageArtifact) GetUncompressedName() string {
+	switch a.CompressionType {
+	case compression.Zstandard, compression.GZip:
+		return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
+	}
+	return a.Path
+}
+
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
+func tarModifierWrapperFunc(ctx *types.Context) func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	return func(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+		// If the destination path already exists I rename target file name with postfix.
+		var destPath string
+
+		// Read data. TODO: We need change archive callback to permit to return a Reader
+		buffer := bytes.Buffer{}
+		if content != nil {
+			if _, err := buffer.ReadFrom(content); err != nil {
+				return nil, nil, err
+			}
+		}
+		tarHash := hashContent(buffer.Bytes())
+
+		// If file is not present on archive but is defined on mods
+		// I receive the callback. Prevent nil exception.
+		if header != nil {
+			switch header.Typeflag {
+			case tar.TypeReg:
+				destPath = filepath.Join(dst, path)
+			default:
+				// Nothing to do. I return original reader
+				return header, buffer.Bytes(), nil
+			}
+
+			existingHash := ""
+			f, err := os.Lstat(destPath)
+			if err == nil {
+				ctx.Debug("File exists already, computing hash for", destPath)
+				hash, herr := hashFileContent(destPath)
+				if herr == nil {
+					existingHash = hash
+				}
+			}
+
+			ctx.Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+			// We want to protect file only if the hash of the files are differing OR the file size are
+			differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
+			// Check if exists
+			if fileHelper.Exists(destPath) && differs {
+				for i := 1; i < 1000; i++ {
+					name := filepath.Join(filepath.Join(filepath.Dir(path),
+						fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
+
+					if fileHelper.Exists(name) {
+						continue
+					}
+					ctx.Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
+						filepath.Join(dst, name)))
+					return &tar.Header{
+						Mode:       header.Mode,
+						Typeflag:   header.Typeflag,
+						PAXRecords: header.PAXRecords,
+						Name:       name,
+					}, buffer.Bytes(), nil
+				}
+			}
+		}
+
+		return header, buffer.Bytes(), nil
+	}
+}
+
+func (a *PackageArtifact) GetProtectFiles(ctx *types.Context) []string {
+	ans := []string{}
+	annotationDir := ""
+
+	if !ctx.Config.ConfigProtectSkip {
+
+		// a.CompileSpec could be nil when artifact.Unpack is used for tree tarball
+		if a.CompileSpec != nil &&
+			a.CompileSpec.GetPackage().HasAnnotation(string(pkg.ConfigProtectAnnnotation)) {
+			dir, ok := a.CompileSpec.GetPackage().GetAnnotations()[string(pkg.ConfigProtectAnnnotation)]
+			if ok {
+				annotationDir = dir
+			}
+		}
+		// TODO: check if skip this if we have a.CompileSpec nil
+
+		cp := config.NewConfigProtect(annotationDir)
+		cp.Map(a.Files, ctx.Config.GetConfigProtectConfFiles())
+
+		// NOTE: for unpack we need files path without initial /
+		ans = cp.GetProtectFiles(false)
+	}
+
+	return ans
+}
+
+// Unpack Untar and decompress (TODO) to the given path
+func (a *PackageArtifact) Unpack(ctx *types.Context, dst string, keepPerms bool) error {
+	if !strings.HasPrefix(dst, "/") {
+		return errors.New("destination must be an absolute path")
+	}
+
+	// Create
+	protectedFiles := a.GetProtectFiles(ctx)
+
+	tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc(ctx))
+
+	switch a.CompressionType {
+	case compression.Zstandard:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+
+		d, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer d.Close()
+
+		_, err = io.Copy(archive, d)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
+			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
+	case compression.GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+
+		_, err = io.Copy(archive, r)
+		if err != nil {
+			return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
+		}
+
+		err = helpers.UntarProtect(a.Path+".uncompressed", dst,
+			ctx.Config.GetGeneral().SameOwner, protectedFiles, tarModifier)
+		if err != nil {
+			return err
+		}
+		return nil
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		return helpers.UntarProtect(a.Path, dst, ctx.Config.GetGeneral().SameOwner,
+			protectedFiles, tarModifier)
+	}
+	return errors.New("Compression type must be supplied")
+}
+
+// FileList generates the list of file of a package from the local archive
+func (a *PackageArtifact) FileList() ([]string, error) {
+	var tr *tar.Reader
+	switch a.CompressionType {
+	case compression.Zstandard:
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := zstd.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+	case compression.GZip:
+		// Create the uncompressed archive
+		archive, err := os.Create(a.Path + ".uncompressed")
+		if err != nil {
+			return []string{}, err
+		}
+		defer os.RemoveAll(a.Path + ".uncompressed")
+		defer archive.Close()
+
+		original, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Cannot open "+a.Path)
+		}
+		defer original.Close()
+
+		bufferedReader := bufio.NewReader(original)
+		r, err := gzip.NewReader(bufferedReader)
+		if err != nil {
+			return []string{}, err
+		}
+		defer r.Close()
+		tr = tar.NewReader(r)
+
+	// Defaults to tar only (covers when "none" is supplied)
+	default:
+		tarFile, err := os.Open(a.Path)
+		if err != nil {
+			return []string{}, errors.Wrap(err, "Could not open package archive")
+		}
+		defer tarFile.Close()
+		tr = tar.NewReader(tarFile)
+
+	}
+
+	var files []string
+	// untar each segment
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return []string{}, err
+		}
+		// determine proper file path info
+		finfo := hdr.FileInfo()
+		fileName := hdr.Name
+		if finfo.Mode().IsDir() {
+			continue
+		}
+		files = append(files, fileName)
+
+		// if a dir, create it, then go to next segment
+	}
+	return files, nil
+}
+
+type CopyJob struct {
+	Src, Dst string
+	Artifact string
+}
+
+func worker(ctx *types.Context, i int, wg *sync.WaitGroup, s <-chan CopyJob) {
+	defer wg.Done()
+
+	for job := range s {
+		_, err := os.Lstat(job.Dst)
+		if err != nil {
+			ctx.Debug("Copying ", job.Src)
+			if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
+				ctx.Warning("Error copying", job, err)
+			}
+		}
+	}
+}
+
+func compileRegexes(regexes []string) []*regexp.Regexp {
+	var result []*regexp.Regexp
+	for _, i := range regexes {
+		r, e := regexp.Compile(i)
+		if e != nil {
+			continue
+		}
+		result = append(result, r)
+	}
+	return result
+}
+
+type ArtifactNode struct {
+	Name string `json:"Name"`
+	Size int    `json:"Size"`
+}
+type ArtifactDiffs struct {
+	Additions []ArtifactNode `json:"Adds"`
+	Deletions []ArtifactNode `json:"Dels"`
+	Changes   []ArtifactNode `json:"Mods"`
+}
+
+type ArtifactLayer struct {
+	FromImage string        `json:"Image1"`
+	ToImage   string        `json:"Image2"`
+	Diffs     ArtifactDiffs `json:"Diff"`
+}
+
+// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
+func ExtractArtifactFromDelta(ctx *types.Context, src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
+
+	archive, err := ctx.Config.GetSystem().TempDir("archive")
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+
+	if strings.HasSuffix(src, ".tar") {
+		rootfs, err := ctx.Config.GetSystem().TempDir("rootfs")
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		defer os.RemoveAll(rootfs) // clean up
+		err = helpers.Untar(src, rootfs, keepPerms)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+		}
+		src = rootfs
+	}
+
+	toCopy := make(chan CopyJob)
+
+	var wg = new(sync.WaitGroup)
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+		go worker(ctx, i, wg, toCopy)
+	}
+
+	// Handle includes in spec. If specified they filter what gets in the package
+
+	if len(includes) > 0 && len(excludes) == 0 {
+		includeRegexp := compileRegexes(includes)
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADDS:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue ADDS
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				ctx.Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				ctx.Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) == 0 && len(excludes) != 0 {
+		excludeRegexp := compileRegexes(excludes)
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		ADD:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range excludeRegexp {
+					if i.MatchString(a.Name) {
+						continue ADD
+					}
+				}
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				ctx.Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				ctx.Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else if len(includes) != 0 && len(excludes) != 0 {
+		includeRegexp := compileRegexes(includes)
+		excludeRegexp := compileRegexes(excludes)
+
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+		EXCLUDES:
+			for _, a := range l.Diffs.Additions {
+				for _, i := range includeRegexp {
+					if i.MatchString(a.Name) {
+						for _, e := range excludeRegexp {
+							if e.MatchString(a.Name) {
+								continue EXCLUDES
+							}
+						}
+						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+						continue EXCLUDES
+					}
+				}
+			}
+			for _, a := range l.Diffs.Changes {
+				ctx.Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				ctx.Debug("File ", a.Name, " deleted")
+			}
+		}
+
+	} else {
+		// Otherwise just grab all
+		for _, l := range layers {
+			// Consider d.Additions (and d.Changes? - warn at least) only
+			for _, a := range l.Diffs.Additions {
+				ctx.Debug("File ", a.Name, " added")
+				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
+			}
+			for _, a := range l.Diffs.Changes {
+				ctx.Debug("File ", a.Name, " changed")
+			}
+			for _, a := range l.Diffs.Deletions {
+				ctx.Debug("File ", a.Name, " deleted")
+			}
+		}
+	}
+
+	close(toCopy)
+	wg.Wait()
+
+	a := NewPackageArtifact(dst)
+	a.CompressionType = t
+	err = a.Compress(archive, concurrency)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error met while creating package archive")
+	}
+	return a, nil
+}

pkg/api/core/types/artifact/artifact_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestArtifact(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Artifact Suite")
+}

pkg/api/core/types/artifact/artifact_test.go

@@ -0,0 +1,272 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+	backend "github.com/mudler/luet/pkg/compiler/backend"
+	compression "github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+
+	. "github.com/mudler/luet/pkg/compiler"
+	helpers "github.com/mudler/luet/pkg/helpers"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Artifact", func() {
+	Context("Simple package build definition", func() {
+		ctx := types.NewContext()
+		It("Generates a verified delta", func() {
+
+			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../../../../tests/fixtures/buildtree")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase(), options.WithContext(types.NewContext()))
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
+			Expect(lspec.Image).To(Equal("luet/base"))
+			Expect(lspec.Seed).To(Equal("alpine"))
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpdir2, err := ioutil.TempDir(os.TempDir(), "tree2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir2) // clean up
+
+			unpacked, err := ioutil.TempDir(os.TempDir(), "unpacked")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(unpacked) // clean up
+
+			rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(rootfs) // clean up
+
+			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM alpine
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin`))
+			b := NewSimpleDockerBackend(ctx)
+			opts := backend.Options{
+				ImageName:      "luet/base",
+				SourcePath:     tmpdir,
+				DockerFileName: "Dockerfile",
+				Destination:    filepath.Join(tmpdir2, "output1.tar"),
+			}
+			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+
+			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(dockerfile).To(Equal(`
+FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
+RUN echo foo > /test
+RUN echo bar > /test2`))
+			opts2 := backend.Options{
+				ImageName:      "test",
+				SourcePath:     tmpdir,
+				DockerFileName: "LuetDockerfile",
+				Destination:    filepath.Join(tmpdir, "output2.tar"),
+			}
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+			diffs, err := compiler.GenerateChanges(ctx, b, opts, opts2)
+			Expect(err).ToNot(HaveOccurred())
+
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
+
+			Expect(diffs).To(Equal(
+				[]ArtifactLayer{{
+					FromImage: "luet/base",
+					ToImage:   "test",
+					Diffs: ArtifactDiffs{
+						Additions: artifacts,
+					},
+				}}))
+			err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			a, err := ExtractArtifactFromDelta(ctx, rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
+			err = helpers.Untar(a.Path, unpacked, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
+			Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
+			content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content1).To(Equal("foo\n"))
+			content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(content2).To(Equal("bar\n"))
+
+			err = a.Hash()
+			Expect(err).ToNot(HaveOccurred())
+			err = a.Verify()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
+
+			err = a.Verify()
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("Generates packages images", func() {
+			b := NewSimpleDockerBackend(ctx)
+			imageprefix := "foo/"
+			testString := []byte(`funky test data`)
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			Expect(os.MkdirAll(filepath.Join(tmpdir, "foo", "bar"), os.ModePerm)).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
+			Expect(err).ToNot(HaveOccurred())
+
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			content, err := ioutil.ReadFile(filepath.Join(result, "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+
+			content, err = ioutil.ReadFile(filepath.Join(result, "foo", "bar", "test"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(content).To(Equal(testString))
+		})
+
+		It("Generates empty packages images", func() {
+			b := NewSimpleDockerBackend(ctx)
+			imageprefix := "foo/"
+
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "artifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpWork, err := ioutil.TempDir(os.TempDir(), "artifact2")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpWork) // clean up
+
+			a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
+
+			err = a.Compress(tmpdir, 1)
+			Expect(err).ToNot(HaveOccurred())
+			resultingImage := imageprefix + "foo--1.0"
+			opts, err := a.GenerateFinalImage(ctx, resultingImage, b, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(opts.ImageName).To(Equal(resultingImage))
+
+			Expect(b.ImageExists(resultingImage)).To(BeTrue())
+
+			result, err := ioutil.TempDir(os.TempDir(), "result")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(result) // clean up
+
+			err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
+			content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(content)).To(Equal(""))
+		})
+
+		It("Retrieves uncompressed name", func() {
+			a := NewPackageArtifact("foo.tar.gz")
+			a.CompressionType = (compression.GZip)
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar.zst")
+			a.CompressionType = compression.Zstandard
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+
+			a = NewPackageArtifact("foo.tar")
+			a.CompressionType = compression.None
+			Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
+		})
+	})
+})

pkg/api/core/types/artifact/cache.go

@@ -0,0 +1,63 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/rancher-sandbox/gofilecache"
+)
+
+type ArtifactCache struct {
+	gofilecache.Cache
+}
+
+func NewCache(dir string) *ArtifactCache {
+	return &ArtifactCache{Cache: *gofilecache.InitCache(dir)}
+}
+
+func (c *ArtifactCache) cacheID(a *PackageArtifact) [64]byte {
+	fingerprint := filepath.Base(a.Path)
+	if a.CompileSpec != nil && a.CompileSpec.Package != nil {
+		fingerprint = a.CompileSpec.Package.GetFingerPrint()
+	}
+	if len(a.Checksums) > 0 {
+		for _, cs := range a.Checksums.List() {
+			t := cs[0]
+			result := cs[1]
+			fingerprint += fmt.Sprintf("+%s:%s", t, result)
+		}
+	}
+	return sha512.Sum512([]byte(fingerprint))
+}
+
+func (c *ArtifactCache) Get(a *PackageArtifact) (string, error) {
+	fileName, _, err := c.Cache.GetFile(c.cacheID(a))
+	return fileName, err
+}
+
+func (c *ArtifactCache) Put(a *PackageArtifact) (gofilecache.OutputID, int64, error) {
+	file, err := os.Open(a.Path)
+	if err != nil {
+		return [64]byte{}, 0, errors.Wrapf(err, "failed opening %s", a.Path)
+	}
+	defer file.Close()
+	return c.Cache.Put(c.cacheID(a), file)
+}

pkg/api/core/types/artifact/cache_test.go

@@ -0,0 +1,90 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cache", func() {
+	Context("CacheID", func() {
+		It("Get and retrieve files", func() {
+			tmpdir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			tmpdirartifact, err := ioutil.TempDir(os.TempDir(), "testartifact")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdirartifact) // clean up
+
+			err = ioutil.WriteFile(filepath.Join(tmpdirartifact, "foo"), []byte(string("foo")), os.ModePerm)
+			Expect(err).ToNot(HaveOccurred())
+
+			a := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			err = a.Compress(tmpdirartifact, 1)
+			Expect(err).ToNot(HaveOccurred())
+
+			cache := NewCache(tmpdir)
+
+			// Put an artifact in the cache and retrieve it later
+			// the artifact is NOT hashed so it is referenced just by the path in the cache
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			path, err := cache.Get(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			b := NewPackageArtifact(path)
+			ctx := types.NewContext()
+			err = b.Unpack(ctx, tmpdir, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "foo"))).To(BeTrue())
+
+			bb, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(string(bb)).To(Equal("foo"))
+
+			// After the artifact is hashed, the fingerprint mutates so the cache doesn't see it hitting again
+			// the test we did above fails as we expect to.
+			a.Hash()
+			_, err = cache.Get(a)
+			Expect(err).To(HaveOccurred())
+
+			a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, _, err = cache.Put(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			c := NewPackageArtifact(filepath.Join(tmpdir, "foo.tar.gz"))
+			c.Hash()
+			c.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "bar"}}
+			_, err = cache.Get(c)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

pkg/api/core/types/artifact/checksum.go

@@ -0,0 +1,91 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact
+
+import (
+
+	//"strconv"
+
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"sort"
+
+	//	. "github.com/mudler/luet/pkg/logger"
+	"github.com/pkg/errors"
+)
+
+type HashImplementation string
+
+const (
+	SHA256 HashImplementation = "sha256"
+)
+
+type Checksums map[string]string
+
+type HashOptions struct {
+	Hasher hash.Hash
+	Type   HashImplementation
+}
+
+func (c Checksums) List() (res [][]string) {
+	keys := make([]string, 0)
+	for k, _ := range c {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		res = append(res, []string{k, c[k]})
+	}
+	return
+}
+
+// Generate generates all Checksums supported for the artifact
+func (c *Checksums) Generate(a *PackageArtifact) error {
+	return c.generateSHA256(a)
+}
+
+func (c Checksums) Compare(d Checksums) error {
+	for t, sum := range d {
+		if v, ok := c[t]; ok && v != sum {
+			return errors.New("Checksum mismsatch")
+		}
+	}
+	return nil
+}
+
+func (c *Checksums) generateSHA256(a *PackageArtifact) error {
+	return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
+}
+
+func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error {
+
+	f, err := os.Open(a.Path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(opts.Hasher, f); err != nil {
+		return err
+	}
+
+	sum := fmt.Sprintf("%x", opts.Hasher.Sum(nil))
+
+	(*c)[string(opts.Type)] = sum
+	return nil
+}

pkg/api/core/types/artifact/checksum_test.go

@@ -0,0 +1,61 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package artifact_test
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "github.com/mudler/luet/pkg/api/core/types/artifact"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Checksum", func() {
+	Context("Generation", func() {
+		It("Compares successfully", func() {
+
+			tmpdir, err := ioutil.TempDir("", "tree")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+			buildsum := Checksums{}
+			definitionsum := Checksums{}
+			definitionsum2 := Checksums{}
+
+			Expect(len(buildsum)).To(Equal(0))
+			Expect(len(definitionsum)).To(Equal(0))
+			Expect(len(definitionsum2)).To(Equal(0))
+
+			err = buildsum.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/build.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			err = definitionsum2.Generate(NewPackageArtifact("../../../../../tests/fixtures/layers/alpine/definition.yaml"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(buildsum)).To(Equal(1))
+			Expect(len(definitionsum)).To(Equal(1))
+			Expect(len(definitionsum2)).To(Equal(1))
+
+			Expect(definitionsum.Compare(buildsum)).To(HaveOccurred())
+			Expect(definitionsum.Compare(definitionsum2)).ToNot(HaveOccurred())
+		})
+	})
+
+})

pkg/api/core/types/config.go

@@ -0,0 +1,472 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//                  Daniele Rondina <geaaru@sabayonlinux.org>
+//             2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/config"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	pkg "github.com/mudler/luet/pkg/package"
+	solver "github.com/mudler/luet/pkg/solver"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+var AvailableResolvers = strings.Join([]string{solver.QLearningResolverType}, " ")
+
+type LuetLoggingConfig struct {
+	// Path of the logfile
+	Path string `mapstructure:"path"`
+	// Enable/Disable logging to file
+	EnableLogFile bool `mapstructure:"enable_logfile"`
+	// Enable JSON format logging in file
+	JsonFormat bool `mapstructure:"json_format"`
+
+	// Log level
+	Level LogLevel `mapstructure:"level"`
+
+	// Enable emoji
+	EnableEmoji bool `mapstructure:"enable_emoji"`
+	// Enable/Disable color in logging
+	Color bool `mapstructure:"color"`
+}
+
+type LuetGeneralConfig struct {
+	SameOwner       bool `yaml:"same_owner,omitempty" mapstructure:"same_owner"`
+	Concurrency     int  `yaml:"concurrency,omitempty" mapstructure:"concurrency"`
+	Debug           bool `yaml:"debug,omitempty" mapstructure:"debug"`
+	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
+	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
+	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+}
+
+type LuetSolverOptions struct {
+	solver.Options `yaml:"options,omitempty"`
+	Type           string            `yaml:"type,omitempty" mapstructure:"type"`
+	LearnRate      float32           `yaml:"rate,omitempty" mapstructure:"rate"`
+	Discount       float32           `yaml:"discount,omitempty" mapstructure:"discount"`
+	MaxAttempts    int               `yaml:"max_attempts,omitempty" mapstructure:"max_attempts"`
+	Implementation solver.SolverType `yaml:"implementation,omitempty" mapstructure:"implementation"`
+}
+
+func (opts LuetSolverOptions) ResolverIsSet() bool {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		return true
+	default:
+		return false
+	}
+}
+
+func (opts LuetSolverOptions) Resolver() solver.PackageResolver {
+	switch opts.Type {
+	case solver.QLearningResolverType:
+		if opts.LearnRate != 0.0 {
+			return solver.NewQLearningResolver(opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+
+		}
+		return solver.SimpleQLearningSolver()
+	}
+
+	return &solver.Explainer{}
+}
+
+func (opts *LuetSolverOptions) CompactString() string {
+	return fmt.Sprintf("type: %s rate: %f, discount: %f, attempts: %d, initialobserved: %d",
+		opts.Type, opts.LearnRate, opts.Discount, opts.MaxAttempts, 999999)
+}
+
+type LuetSystemConfig struct {
+	DatabaseEngine string `yaml:"database_engine" mapstructure:"database_engine"`
+	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
+	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
+	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
+}
+
+func (s *LuetSystemConfig) SetRootFS(path string) error {
+	p, err := fileHelper.Rel2Abs(path)
+	if err != nil {
+		return err
+	}
+
+	s.Rootfs = p
+	return nil
+}
+
+func (sc *LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
+	dbpath := filepath.Join(sc.Rootfs, sc.DatabasePath)
+	dbpath = filepath.Join(dbpath, "repos/"+name)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemRepoDatabaseDirPath() string {
+	dbpath := filepath.Join(sc.Rootfs,
+		sc.DatabasePath)
+	err := os.MkdirAll(dbpath, os.ModePerm)
+	if err != nil {
+		panic(err)
+	}
+	return dbpath
+}
+
+func (sc *LuetSystemConfig) GetSystemPkgsCacheDirPath() (ans string) {
+	var cachepath string
+	if sc.PkgsCachePath != "" {
+		cachepath = sc.PkgsCachePath
+	} else {
+		// Create dynamic cache for test suites
+		cachepath, _ = ioutil.TempDir(os.TempDir(), "cachepkgs")
+	}
+
+	if filepath.IsAbs(cachepath) {
+		ans = cachepath
+	} else {
+		ans = filepath.Join(sc.GetSystemRepoDatabaseDirPath(), cachepath)
+	}
+
+	return
+}
+
+func (sc *LuetSystemConfig) GetRootFsAbs() (string, error) {
+	return filepath.Abs(sc.Rootfs)
+}
+
+type LuetKV struct {
+	Key   string `json:"key" yaml:"key" mapstructure:"key"`
+	Value string `json:"value" yaml:"value" mapstructure:"value"`
+}
+
+type LuetConfig struct {
+	Logging LuetLoggingConfig `yaml:"logging,omitempty" mapstructure:"logging"`
+	General LuetGeneralConfig `yaml:"general,omitempty" mapstructure:"general"`
+	System  LuetSystemConfig  `yaml:"system" mapstructure:"system"`
+	Solver  LuetSolverOptions `yaml:"solver,omitempty" mapstructure:"solver"`
+
+	RepositoriesConfDir  []string         `yaml:"repos_confdir,omitempty" mapstructure:"repos_confdir"`
+	ConfigProtectConfDir []string         `yaml:"config_protect_confdir,omitempty" mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `yaml:"config_protect_skip,omitempty" mapstructure:"config_protect_skip"`
+	ConfigFromHost       bool             `yaml:"config_from_host,omitempty" mapstructure:"config_from_host"`
+	SystemRepositories   LuetRepositories `yaml:"repositories,omitempty" mapstructure:"repositories"`
+
+	FinalizerEnvs []LuetKV `json:"finalizer_envs,omitempty" yaml:"finalizer_envs,omitempty" mapstructure:"finalizer_envs,omitempty"`
+
+	ConfigProtectConfFiles []config.ConfigProtectConfFile `yaml:"-" mapstructure:"-"`
+}
+
+func (c *LuetConfig) GetSystemDB() pkg.PackageDatabase {
+	switch c.GetSystem().DatabaseEngine {
+	case "boltdb":
+		return pkg.NewBoltDatabase(
+			filepath.Join(c.GetSystem().GetSystemRepoDatabaseDirPath(), "luet.db"))
+	default:
+		return pkg.NewInMemoryDatabase(true)
+	}
+}
+
+func (c *LuetConfig) AddSystemRepository(r LuetRepository) {
+	c.SystemRepositories = append(c.SystemRepositories, r)
+}
+
+func (c *LuetConfig) GetFinalizerEnvsMap() map[string]string {
+	ans := make(map[string]string)
+
+	for _, kv := range c.FinalizerEnvs {
+		ans[kv.Key] = kv.Value
+	}
+	return ans
+}
+
+func (c *LuetConfig) SetFinalizerEnv(k, v string) {
+	keyPresent := false
+	envs := []LuetKV{}
+
+	for _, kv := range c.FinalizerEnvs {
+		if kv.Key == k {
+			keyPresent = true
+			envs = append(envs, LuetKV{Key: kv.Key, Value: v})
+		} else {
+			envs = append(envs, kv)
+		}
+	}
+	if !keyPresent {
+		envs = append(envs, LuetKV{Key: k, Value: v})
+	}
+
+	c.FinalizerEnvs = envs
+}
+
+func (c *LuetConfig) GetFinalizerEnvs() []string {
+	ans := []string{}
+	for _, kv := range c.FinalizerEnvs {
+		ans = append(ans, fmt.Sprintf("%s=%s", kv.Key, kv.Value))
+	}
+	return ans
+}
+
+func (c *LuetConfig) GetFinalizerEnv(k string) (string, error) {
+	keyNotPresent := true
+	ans := ""
+	for _, kv := range c.FinalizerEnvs {
+		if kv.Key == k {
+			keyNotPresent = false
+			ans = kv.Value
+		}
+	}
+
+	if keyNotPresent {
+		return "", errors.New("Finalizer key " + k + " not found")
+	}
+	return ans, nil
+}
+
+func (c *LuetConfig) GetLogging() *LuetLoggingConfig {
+	return &c.Logging
+}
+
+func (c *LuetConfig) GetGeneral() *LuetGeneralConfig {
+	return &c.General
+}
+
+func (c *LuetConfig) GetSystem() *LuetSystemConfig {
+	return &c.System
+}
+
+func (c *LuetConfig) GetSolverOptions() *LuetSolverOptions {
+	return &c.Solver
+}
+
+func (c *LuetConfig) YAML() ([]byte, error) {
+	return yaml.Marshal(c)
+}
+
+func (c *LuetConfig) GetConfigProtectConfFiles() []config.ConfigProtectConfFile {
+	return c.ConfigProtectConfFiles
+}
+
+func (c *LuetConfig) AddConfigProtectConfFile(file *config.ConfigProtectConfFile) {
+	if c.ConfigProtectConfFiles == nil {
+		c.ConfigProtectConfFiles = []config.ConfigProtectConfFile{*file}
+	} else {
+		c.ConfigProtectConfFiles = append(c.ConfigProtectConfFiles, *file)
+	}
+}
+
+func (c *LuetConfig) LoadRepositories(ctx *Context) error {
+	var regexRepo = regexp.MustCompile(`.yml$|.yaml$`)
+	var err error
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, rdir := range c.RepositoriesConfDir {
+
+		rdir = filepath.Join(rootfs, rdir)
+
+		ctx.Debug("Parsing Repository Directory", rdir, "...")
+
+		files, err := ioutil.ReadDir(rdir)
+		if err != nil {
+			ctx.Debug("Skip dir", rdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexRepo.MatchString(file.Name()) {
+				ctx.Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(rdir, file.Name()))
+			if err != nil {
+				ctx.Warning("On read file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := LoadRepository(content)
+			if err != nil {
+				ctx.Warning("On parse file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Urls) == 0 || r.Type == "" {
+				ctx.Warning("Invalid repository ", file.Name())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddSystemRepository(*r)
+		}
+	}
+	return nil
+}
+
+func (c *LuetConfig) GetSystemRepository(name string) (*LuetRepository, error) {
+	var ans *LuetRepository = nil
+
+	for idx, repo := range c.SystemRepositories {
+		if repo.Name == name {
+			ans = &c.SystemRepositories[idx]
+			break
+		}
+	}
+	if ans == nil {
+		return nil, errors.New("Repository " + name + " not found")
+	}
+
+	return ans, nil
+}
+
+func (c *LuetConfig) LoadConfigProtect(ctx *Context) error {
+	var regexConfs = regexp.MustCompile(`.yml$`)
+	var err error
+
+	rootfs := ""
+
+	// Respect the rootfs param on read repositories
+	if !c.ConfigFromHost {
+		rootfs, err = c.GetSystem().GetRootFsAbs()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, cdir := range c.ConfigProtectConfDir {
+		cdir = filepath.Join(rootfs, cdir)
+
+		ctx.Debug("Parsing Config Protect Directory", cdir, "...")
+
+		files, err := ioutil.ReadDir(cdir)
+		if err != nil {
+			ctx.Debug("Skip dir", cdir, ":", err.Error())
+			continue
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				continue
+			}
+
+			if !regexConfs.MatchString(file.Name()) {
+				ctx.Debug("File", file.Name(), "skipped.")
+				continue
+			}
+
+			content, err := ioutil.ReadFile(path.Join(cdir, file.Name()))
+			if err != nil {
+				ctx.Warning("On read file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			r, err := loadConfigProtectConFile(file.Name(), content)
+			if err != nil {
+				ctx.Warning("On parse file", file.Name(), ":", err.Error())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			if r.Name == "" || len(r.Directories) == 0 {
+				ctx.Warning("Invalid config protect file", file.Name())
+				ctx.Warning("File", file.Name(), "skipped.")
+				continue
+			}
+
+			c.AddConfigProtectConfFile(r)
+		}
+	}
+	return nil
+
+}
+
+func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProtectConfFile, error) {
+	ans := config.NewConfigProtectConfFile(filename)
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func (c *LuetLoggingConfig) SetLogLevel(s LogLevel) {
+	c.Level = s
+}
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !filepath.IsAbs(c.TmpDirBase) {
+		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
+		if err != nil {
+			return errors.Wrap(err, "while converting relative path to absolute path")
+		}
+		c.TmpDirBase = abs
+	}
+
+	if _, err := os.Stat(c.TmpDirBase); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/api/core/types/config_suite_test.go

@@ -0,0 +1,28 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestAPITypes(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Types Suite")
+}

pkg/api/core/types/config_test.go

@@ -0,0 +1,89 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+	Context("Load Repository1", func() {
+		ctx := types.NewContext()
+
+		ctx.Config.RepositoriesConfDir = []string{
+			"../../../../tests/fixtures/repos.conf.d",
+		}
+		err := ctx.Config.LoadRepositories(ctx)
+
+		It("Check Load Repository 1", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
+			Expect(ctx.Config.SystemRepositories[0].Name).Should(Equal("test1"))
+			Expect(ctx.Config.SystemRepositories[0].Priority).Should(Equal(999))
+			Expect(ctx.Config.SystemRepositories[0].Type).Should(Equal("disk"))
+			Expect(len(ctx.Config.SystemRepositories[0].Urls)).Should(Equal(1))
+			Expect(ctx.Config.SystemRepositories[0].Urls[0]).Should(Equal("tests/repos/test1"))
+		})
+
+		It("Chec Load Repository 2", func() {
+			Expect(err).Should(BeNil())
+			Expect(len(ctx.Config.SystemRepositories)).Should(Equal(2))
+			Expect(ctx.Config.SystemRepositories[1].Name).Should(Equal("test2"))
+			Expect(ctx.Config.SystemRepositories[1].Priority).Should(Equal(1000))
+			Expect(ctx.Config.SystemRepositories[1].Type).Should(Equal("disk"))
+			Expect(len(ctx.Config.SystemRepositories[1].Urls)).Should(Equal(1))
+			Expect(ctx.Config.SystemRepositories[1].Urls[0]).Should(Equal("tests/repos/test2"))
+		})
+	})
+	Context("Simple temporary directory creation", func() {
+
+		It("Create Temporary directory", func() {
+			ctx := types.NewContext()
+
+			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
+
+			tmpDir, err := ctx.Config.GetSystem().TempDir("test1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(fileHelper.Exists(tmpDir)).To(BeTrue())
+
+			defer os.RemoveAll(tmpDir)
+		})
+
+		It("Create Temporary file", func() {
+			ctx := types.NewContext()
+
+			ctx.Config.GetSystem().TmpDirBase = os.TempDir() + "/tmpluet"
+
+			tmpFile, err := ctx.Config.GetSystem().TempFile("testfile1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(fileHelper.Exists(tmpFile.Name())).To(BeTrue())
+
+			defer os.Remove(tmpFile.Name())
+		})
+
+	})
+
+})

pkg/api/core/types/context.go

@@ -0,0 +1,416 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/kyokomi/emoji"
+	"github.com/mudler/luet/pkg/helpers/terminal"
+	"github.com/pkg/errors"
+	"github.com/pterm/pterm"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/term"
+)
+
+const (
+	ErrorLevel   LogLevel = "error"
+	WarningLevel LogLevel = "warning"
+	InfoLevel    LogLevel = "info"
+	SuccessLevel LogLevel = "success"
+	FatalLevel   LogLevel = "fatal"
+)
+
+type Context struct {
+	context.Context
+	Config     *LuetConfig
+	IsTerminal bool
+	NoSpinner  bool
+
+	s           *pterm.SpinnerPrinter
+	spinnerLock *sync.Mutex
+	z           *zap.Logger
+	AreaPrinter *pterm.AreaPrinter
+	ProgressBar *pterm.ProgressbarPrinter
+}
+
+func NewContext() *Context {
+	return &Context{
+		spinnerLock: &sync.Mutex{},
+		IsTerminal:  terminal.IsTerminal(os.Stdout),
+		Config: &LuetConfig{
+			ConfigFromHost: true,
+			Logging:        LuetLoggingConfig{},
+			General:        LuetGeneralConfig{},
+			System: LuetSystemConfig{
+				DatabasePath: filepath.Join("var", "db", "packages"),
+				TmpDirBase:   filepath.Join(os.TempDir(), "tmpluet")},
+			Solver: LuetSolverOptions{},
+		},
+		s: pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
+	}
+}
+
+func (c *Context) Copy() *Context {
+
+	configCopy := *c.Config
+	configCopy.System = *c.Config.GetSystem()
+	configCopy.General = *c.Config.GetGeneral()
+	configCopy.Logging = *c.Config.GetLogging()
+
+	ctx := *c
+	ctxCopy := &ctx
+	ctxCopy.Config = &configCopy
+
+	return ctxCopy
+}
+
+// GetTerminalSize returns the width and the height of the active terminal.
+func (c *Context) GetTerminalSize() (width, height int, err error) {
+	w, h, err := term.GetSize(int(os.Stdout.Fd()))
+	if w <= 0 {
+		w = 0
+	}
+	if h <= 0 {
+		h = 0
+	}
+	if err != nil {
+		err = errors.New("size not detectable")
+	}
+	return w, h, err
+}
+
+func (c *Context) Init() (err error) {
+	if c.IsTerminal {
+		if !c.Config.Logging.Color {
+			c.Debug("Disabling colors")
+			c.NoColor()
+		}
+	} else {
+		c.Debug("Not a terminal, disabling colors")
+		c.NoColor()
+	}
+
+	c.Debug("Colors", c.Config.GetLogging().Color)
+	c.Debug("Logging level", c.Config.GetLogging().Level)
+	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
+
+	if c.Config.GetLogging().EnableLogFile && c.Config.GetLogging().Path != "" {
+		// Init zap logger
+		err = c.InitZap()
+		if err != nil {
+			return
+		}
+	}
+
+	// Load repositories
+	err = c.Config.LoadRepositories(c)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (c *Context) NoColor() {
+	pterm.DisableColor()
+}
+
+func (c *Context) Ask() bool {
+	var input string
+
+	c.Info("Do you want to continue with this operation? [y/N]: ")
+	_, err := fmt.Scanln(&input)
+	if err != nil {
+		return false
+	}
+	input = strings.ToLower(input)
+
+	if input == "y" || input == "yes" {
+		return true
+	}
+	return false
+}
+
+func (c *Context) InitZap() error {
+	var err error
+	if c.z == nil {
+		// TODO: test permission for open logfile.
+		cfg := zap.NewProductionConfig()
+		cfg.OutputPaths = []string{c.Config.GetLogging().Path}
+		cfg.Level = c.Config.GetLogging().Level.ZapLevel()
+		cfg.ErrorOutputPaths = []string{}
+		if c.Config.GetLogging().JsonFormat {
+			cfg.Encoding = "json"
+		} else {
+			cfg.Encoding = "console"
+		}
+		cfg.DisableCaller = true
+		cfg.DisableStacktrace = true
+		cfg.EncoderConfig.TimeKey = "time"
+		cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+		c.z, err = cfg.Build()
+		if err != nil {
+			fmt.Fprint(os.Stderr, "Error on initialize file logger: "+err.Error()+"\n")
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Spinner starts the spinner
+func (c *Context) Spinner() {
+	if !c.IsTerminal || c.NoSpinner {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	var confLevel int
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	if 2 > confLevel {
+		return
+	}
+
+	if !c.s.IsActive {
+		c.s, _ = c.s.Start()
+	}
+}
+
+func (c *Context) Screen(text string) {
+	pterm.DefaultHeader.WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(2).Println(text)
+	//pterm.DefaultCenter.Print(pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Sprint(text))
+}
+
+func (c *Context) SpinnerText(suffix, prefix string) {
+	if !c.IsTerminal || c.NoSpinner {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	if c.Config.GetGeneral().Debug {
+		fmt.Printf("%s %s\n",
+			suffix, prefix,
+		)
+	} else {
+		c.s.UpdateText(suffix + prefix)
+	}
+}
+
+func (c *Context) SpinnerStop() {
+	if !c.IsTerminal {
+		return
+	}
+
+	c.spinnerLock.Lock()
+	defer c.spinnerLock.Unlock()
+	var confLevel int
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	if 2 > confLevel {
+		return
+	}
+	if c.s != nil {
+		c.s.Success()
+	}
+}
+
+func (c *Context) log2File(level LogLevel, msg string) {
+	switch level {
+	case FatalLevel:
+		c.z.Fatal(msg)
+	case ErrorLevel:
+		c.z.Error(msg)
+	case WarningLevel:
+		c.z.Warn(msg)
+	case InfoLevel, SuccessLevel:
+		c.z.Info(msg)
+	default:
+		c.z.Debug(msg)
+	}
+}
+
+func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
+	var message string
+	var confLevel, msgLevel int
+
+	if c.Config.GetGeneral().Debug {
+		confLevel = 3
+		pterm.EnableDebugMessages()
+	} else {
+		confLevel = c.Config.GetLogging().Level.ToNumber()
+	}
+	msgLevel = level.ToNumber()
+
+	if msgLevel > confLevel {
+		return
+	}
+
+	for _, m := range msg {
+		message += " " + fmt.Sprintf("%v", m)
+	}
+
+	// Color message
+	levelMsg := message
+
+	if c.Config.GetLogging().Color {
+		switch level {
+		case WarningLevel:
+			levelMsg = pterm.LightYellow(":construction: warning" + message)
+		case InfoLevel:
+			levelMsg = message
+		case SuccessLevel:
+			levelMsg = pterm.LightGreen(message)
+		case ErrorLevel:
+			levelMsg = pterm.Red(message)
+		default:
+			levelMsg = pterm.Blue(message)
+		}
+	}
+
+	// Strip emoji if needed
+	if c.Config.GetLogging().EnableEmoji && c.IsTerminal {
+		levelMsg = emoji.Sprint(levelMsg)
+	} else {
+		re := regexp.MustCompile(`[:][\w]+[:]`)
+		levelMsg = re.ReplaceAllString(levelMsg, "")
+	}
+
+	if c.z != nil {
+		c.log2File(level, message)
+	}
+
+	// Print the message based on the level
+	switch level {
+	case SuccessLevel:
+		if ln {
+			pterm.Success.Println(levelMsg)
+		} else {
+			pterm.Success.Print(levelMsg)
+		}
+	case InfoLevel:
+		if ln {
+			pterm.Info.Println(levelMsg)
+		} else {
+			pterm.Info.Print(levelMsg)
+		}
+	case WarningLevel:
+		if ln {
+			pterm.Warning.Println(levelMsg)
+		} else {
+			pterm.Warning.Print(levelMsg)
+		}
+	case ErrorLevel:
+		if ln {
+			pterm.Error.Println(levelMsg)
+		} else {
+			pterm.Error.Print(levelMsg)
+		}
+	case FatalLevel:
+		if ln {
+			pterm.Fatal.Println(levelMsg)
+		} else {
+			pterm.Fatal.Print(levelMsg)
+		}
+	default:
+		if ln {
+			pterm.Debug.Println(levelMsg)
+		} else {
+			pterm.Debug.Print(levelMsg)
+		}
+	}
+}
+
+func (c *Context) Warning(mess ...interface{}) {
+	c.Msg("warning", true, mess...)
+	if c.Config.GetGeneral().FatalWarns {
+		os.Exit(2)
+	}
+}
+
+func (c *Context) Debug(mess ...interface{}) {
+	pc, file, line, ok := runtime.Caller(1)
+	if ok {
+		mess = append([]interface{}{fmt.Sprintf("(%s:#%d:%v)",
+			path.Base(file), line, runtime.FuncForPC(pc).Name())}, mess...)
+	}
+	c.Msg("debug", true, mess...)
+}
+
+func (c *Context) Info(mess ...interface{}) {
+	c.Msg("info", true, mess...)
+}
+
+func (c *Context) Success(mess ...interface{}) {
+	c.Msg("success", true, mess...)
+}
+
+func (c *Context) Error(mess ...interface{}) {
+	c.Msg("error", true, mess...)
+}
+
+func (c *Context) Fatal(mess ...interface{}) {
+	c.Error(mess...)
+	os.Exit(1)
+}
+
+type LogLevel string
+
+func (level LogLevel) ToNumber() int {
+	switch level {
+	case ErrorLevel, FatalLevel:
+		return 0
+	case WarningLevel:
+		return 1
+	case InfoLevel, SuccessLevel:
+		return 2
+	default: // debug
+		return 3
+	}
+}
+
+func (level LogLevel) ZapLevel() zap.AtomicLevel {
+	switch level {
+	case FatalLevel:
+		return zap.NewAtomicLevelAt(zap.FatalLevel)
+	case ErrorLevel:
+		return zap.NewAtomicLevelAt(zap.ErrorLevel)
+	case WarningLevel:
+		return zap.NewAtomicLevelAt(zap.WarnLevel)
+	case InfoLevel, SuccessLevel:
+		return zap.NewAtomicLevelAt(zap.InfoLevel)
+	default:
+		return zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+}

pkg/api/core/types/repository.go

@@ -0,0 +1,96 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"fmt"
+	"runtime"
+
+	"gopkg.in/yaml.v2"
+)
+
+type LuetRepository struct {
+	Name           string            `json:"name" yaml:"name" mapstructure:"name"`
+	Description    string            `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description"`
+	Urls           []string          `json:"urls" yaml:"urls" mapstructure:"urls"`
+	Type           string            `json:"type" yaml:"type" mapstructure:"type"`
+	Mode           string            `json:"mode,omitempty" yaml:"mode,omitempty" mapstructure:"mode,omitempty"`
+	Priority       int               `json:"priority,omitempty" yaml:"priority,omitempty" mapstructure:"priority"`
+	Enable         bool              `json:"enable" yaml:"enable" mapstructure:"enable"`
+	Cached         bool              `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"`
+	Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"`
+	TreePath       string            `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"`
+	MetaPath       string            `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"`
+	Verify         bool              `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"`
+	Arch           string            `json:"arch,omitempty" yaml:"arch,omitempty" mapstructure:"arch"`
+
+	ReferenceID string `json:"reference,omitempty" yaml:"reference,omitempty" mapstructure:"reference"`
+
+	// Incremented value that identify revision of the repository in a user-friendly way.
+	Revision int `json:"revision,omitempty" yaml:"-" mapstructure:"-"`
+	// Epoch time in seconds
+	LastUpdate string `json:"last_update,omitempty" yaml:"-" mapstructure:"-"`
+}
+
+func (r *LuetRepository) String() string {
+	return fmt.Sprintf("[%s] prio: %d, type: %s, enable: %t, cached: %t",
+		r.Name, r.Priority, r.Type, r.Enable, r.Cached)
+}
+
+// Enabled returns a boolean indicating if the repository should be considered enabled or not
+func (r *LuetRepository) Enabled() bool {
+	return r.Arch != "" && r.Arch == runtime.GOARCH && !r.Enable || r.Enable
+}
+
+type LuetRepositories []LuetRepository
+
+func (l LuetRepositories) Enabled() (res LuetRepositories) {
+	for _, r := range l {
+		if r.Enabled() {
+			res = append(res, r)
+		}
+	}
+	return
+}
+
+func NewLuetRepository(name, t, descr string, urls []string, priority int, enable, cached bool) *LuetRepository {
+	return &LuetRepository{
+		Name:           name,
+		Description:    descr,
+		Urls:           urls,
+		Type:           t,
+		Priority:       priority,
+		Enable:         enable,
+		Cached:         cached,
+		Authentication: make(map[string]string),
+	}
+}
+
+func NewEmptyLuetRepository() *LuetRepository {
+	return &LuetRepository{
+		Priority:       9999,
+		Authentication: make(map[string]string),
+	}
+}
+
+func LoadRepository(data []byte) (*LuetRepository, error) {
+	ans := NewEmptyLuetRepository()
+	err := yaml.Unmarshal(data, &ans)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}

pkg/api/core/types/repository_test.go

@@ -0,0 +1,49 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types_test
+
+import (
+	"runtime"
+
+	types "github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Types", func() {
+	Context("Repository detects underlying arch", func() {
+		It("is enabled if arch is matching", func() {
+			r := types.LuetRepository{Arch: runtime.GOARCH}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("is disabled if arch is NOT matching", func() {
+			r := types.LuetRepository{Arch: "foo"}
+			Expect(r.Enabled()).To(BeFalse())
+		})
+		It("is enabled if arch is NOT matching and enabled is true", func() {
+			r := types.LuetRepository{Arch: "foo", Enable: true}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("enabled is true", func() {
+			r := types.LuetRepository{Enable: true}
+			Expect(r.Enabled()).To(BeTrue())
+		})
+		It("enabled is false", func() {
+			r := types.LuetRepository{Enable: false}
+			Expect(r.Enabled()).To(BeFalse())
+		})
+	})
+})

pkg/box/exec.go

@@ -0,0 +1,185 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	b64 "encoding/base64"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
+	"github.com/pkg/errors"
+)
+
+type Box interface {
+	Run() error
+	Exec() error
+}
+
+type DefaultBox struct {
+	Name                  string
+	Root                  string
+	Env                   []string
+	Cmd                   string
+	Args                  []string
+	HostMounts            []string
+	Stdin, Stdout, Stderr bool
+}
+
+func NewBox(cmd string, args, hostmounts, env []string, rootfs string, stdin, stdout, stderr bool) Box {
+	return &DefaultBox{
+		Stdin:      stdin,
+		Stdout:     stdout,
+		Stderr:     stderr,
+		Cmd:        cmd,
+		Args:       args,
+		Root:       rootfs,
+		HostMounts: hostmounts,
+		Env:        env,
+	}
+}
+
+func (b *DefaultBox) Exec() error {
+
+	if err := mountProc(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting proc on rootfs")
+	}
+	if err := mountDev(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting dev on rootfs")
+	}
+
+	for _, hostMount := range b.HostMounts {
+		target := hostMount
+		if strings.Contains(hostMount, ":") {
+			dest := strings.Split(hostMount, ":")
+			if len(dest) != 2 {
+				return errors.New("Invalid arguments for mount, it can be: fullpath, or source:target")
+			}
+			hostMount = dest[0]
+			target = dest[1]
+		}
+		if err := mountBind(hostMount, b.Root, target); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed mounting %s on rootfs", hostMount))
+		}
+	}
+
+	if err := PivotRoot(b.Root); err != nil {
+		return errors.Wrap(err, "Failed switching pivot on rootfs")
+	}
+	cmd := exec.Command(b.Cmd, b.Args...)
+
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+
+	cmd.Env = b.Env
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error running the %s command in box.Exec", b.Cmd))
+	}
+	return nil
+}
+
+func (b *DefaultBox) Run() error {
+
+	if !fileHelper.Exists(b.Root) {
+		return errors.New(b.Root + " does not exist")
+	}
+
+	// This matches with exec CLI command in luet
+	// TODO: Pass by env var as well
+	execCmd := []string{"exec", "--rootfs", b.Root, "--entrypoint", b.Cmd}
+
+	if b.Stdin {
+		execCmd = append(execCmd, "--stdin")
+	}
+
+	if b.Stderr {
+		execCmd = append(execCmd, "--stderr")
+	}
+
+	if b.Stdout {
+		execCmd = append(execCmd, "--stdout")
+	}
+	// Encode the command in base64 to avoid bad input from the args given
+	execCmd = append(execCmd, "--decode")
+
+	for _, m := range b.HostMounts {
+		execCmd = append(execCmd, "--mount")
+		execCmd = append(execCmd, m)
+	}
+
+	for _, e := range b.Env {
+		execCmd = append(execCmd, "--env")
+		execCmd = append(execCmd, e)
+	}
+
+	for _, a := range b.Args {
+		execCmd = append(execCmd, b64.StdEncoding.EncodeToString([]byte(a)))
+	}
+
+	cmd := exec.Command("/proc/self/exe", execCmd...)
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS |
+			syscall.CLONE_NEWUTS |
+			syscall.CLONE_NEWIPC |
+			syscall.CLONE_NEWPID |
+			syscall.CLONE_NEWNET |
+			syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getuid(),
+				Size:        1,
+			},
+		},
+		GidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getgid(),
+				Size:        1,
+			},
+		},
+	}
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "Failed running Box command in box.Run")
+	}
+	return nil
+}

pkg/box/rootfs.go

@@ -0,0 +1,94 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func PivotRoot(newroot string) error {
+	putold := filepath.Join(newroot, "/.pivot_root")
+
+	// bind mount newroot to itself - this is a slight hack needed to satisfy the
+	// pivot_root requirement that newroot and putold must not be on the same
+	// filesystem as the current root
+	if err := syscall.Mount(newroot, newroot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
+		return err
+	}
+
+	// create putold directory
+	if err := os.MkdirAll(putold, 0700); err != nil {
+		return err
+	}
+
+	// call pivot_root
+	if err := syscall.PivotRoot(newroot, putold); err != nil {
+		return err
+	}
+
+	// ensure current working directory is set to new root
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	// umount putold, which now lives at /.pivot_root
+	putold = "/.pivot_root"
+	if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil {
+		return err
+	}
+
+	// remove putold
+	if err := os.RemoveAll(putold); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountProc(newroot string) error {
+	source := "proc"
+	target := filepath.Join(newroot, "/proc")
+	fstype := "proc"
+	flags := 0
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, uintptr(flags), data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountBind(hostfolder, newroot, dst string) error {
+	source := hostfolder
+	target := filepath.Join(newroot, dst)
+	fstype := "bind"
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, syscall.MS_BIND|syscall.MS_REC, data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountDev(newroot string) error {
+	return mountBind("/dev", newroot, "/dev")
+}

pkg/bus/events.go

@@ -0,0 +1,103 @@
+package bus
+
+import (
+	"github.com/mudler/go-pluggable"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	// Package events
+
+	// EventPackageInstall is the event fired when a new package is being installed
+	EventPackageInstall pluggable.EventType = "package.install"
+	// EventPackageUnInstall is the event fired when a new package is being uninstalled
+	EventPackageUnInstall pluggable.EventType = "package.uninstall"
+
+	// Package build
+
+	// EventPackagePreBuild is the event fired before a package is being built
+	EventPackagePreBuild pluggable.EventType = "package.pre.build"
+	// EventPackagePreBuildArtifact is the event fired before a package tarball is being generated
+	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
+	// EventPackagePostBuildArtifact is the event fired after a package tarball is generated
+	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
+	// EventPackagePostBuild is the event fired after a package was built
+	EventPackagePostBuild pluggable.EventType = "package.post.build"
+
+	// Image build
+
+	// EventImagePreBuild is the event fired before a image is being built
+	EventImagePreBuild pluggable.EventType = "image.pre.build"
+	// EventImagePrePull is the event fired before a image is being pulled
+	EventImagePrePull pluggable.EventType = "image.pre.pull"
+	// EventImagePrePush is the event fired before a image is being pushed
+	EventImagePrePush pluggable.EventType = "image.pre.push"
+	// EventImagePostBuild is the event fired after an image is being built
+	EventImagePostBuild pluggable.EventType = "image.post.build"
+	// EventImagePostPull is the event fired after an image is being pulled
+	EventImagePostPull pluggable.EventType = "image.post.pull"
+	// EventImagePostPush is the event fired after an image is being pushed
+	EventImagePostPush pluggable.EventType = "image.post.push"
+
+	// Repository events
+
+	// EventRepositoryPreBuild is the event fired before a repository is being built
+	EventRepositoryPreBuild pluggable.EventType = "repository.pre.build"
+	// EventRepositoryPostBuild is the event fired after a repository was built
+	EventRepositoryPostBuild pluggable.EventType = "repository.post.build"
+
+	// Image unpack
+
+	// EventImagePreUnPack is the event fired before unpacking an image to a local dir
+	EventImagePreUnPack pluggable.EventType = "image.pre.unpack"
+	// EventImagePostUnPack is the event fired after unpacking an image to a local dir
+	EventImagePostUnPack pluggable.EventType = "image.post.unpack"
+)
+
+// Manager is the bus instance manager, which subscribes plugins to events emitted by Luet
+var Manager *Bus = &Bus{
+	Manager: pluggable.NewManager(
+		[]pluggable.EventType{
+			EventPackageInstall,
+			EventPackageUnInstall,
+			EventPackagePreBuild,
+			EventPackagePreBuildArtifact,
+			EventPackagePostBuildArtifact,
+			EventPackagePostBuild,
+			EventRepositoryPreBuild,
+			EventRepositoryPostBuild,
+			EventImagePreBuild,
+			EventImagePrePull,
+			EventImagePrePush,
+			EventImagePostBuild,
+			EventImagePostPull,
+			EventImagePostPush,
+			EventImagePreUnPack,
+			EventImagePostUnPack,
+		},
+	),
+}
+
+type Bus struct {
+	*pluggable.Manager
+}
+
+func (b *Bus) Initialize(plugin ...string) {
+	b.Manager.Load(plugin...).Register()
+
+	for _, e := range b.Manager.Events {
+		b.Manager.Response(e, func(p *pluggable.Plugin, r *pluggable.EventResponse) {
+			if r.Errored() {
+				logrus.Fatal("Plugin", p.Name, "at", p.Executable, "Error", r.Error)
+			}
+			logrus.Debug(
+				"plugin_event",
+				"received from",
+				p.Name,
+				"at",
+				p.Executable,
+				r,
+			)
+		})
+	}
+}

pkg/compiler/artifact.go

@@ -1,175 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-
-	. "github.com/mudler/luet/pkg/logger"
-	"github.com/mudler/luet/pkg/solver"
-
-	"github.com/mudler/luet/pkg/helpers"
-	"github.com/pkg/errors"
-)
-
-type PackageArtifact struct {
-	Path         string
-	Dependencies []Artifact
-	CompileSpec  CompilationSpec
-
-	SourceAssertion solver.PackagesAssertions
-}
-
-func NewPackageArtifact(path string) Artifact {
-	return &PackageArtifact{Path: path, Dependencies: []Artifact{}}
-}
-
-func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
-}
-
-func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
-	a.CompileSpec = as
-}
-
-func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
-	return a.CompileSpec
-}
-
-func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
-}
-
-func (a *PackageArtifact) GetDependencies() []Artifact {
-	return a.Dependencies
-}
-
-func (a *PackageArtifact) SetDependencies(d []Artifact) {
-	a.Dependencies = d
-}
-
-func (a *PackageArtifact) GetPath() string {
-	return a.Path
-}
-
-func (a *PackageArtifact) SetPath(p string) {
-	a.Path = p
-}
-
-type CopyJob struct {
-	Src, Dst string
-	Artifact string
-}
-
-func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
-	defer wg.Done()
-
-	for job := range s {
-		Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-			err = helpers.CopyDir(job.Src, job.Dst)
-			if err != nil {
-				Fatal("Error copying dir", job, err)
-			}
-			continue
-		}
-
-		if !helpers.Exists(job.Dst) {
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
-				Fatal("Error copying", job, err)
-			}
-		}
-	}
-}
-
-// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
-func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string) (Artifact, error) {
-
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
-	}
-	defer os.RemoveAll(archive) // clean up
-
-	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-		}
-		defer os.RemoveAll(rootfs) // clean up
-		err = helpers.Untar(src, rootfs, keepPerms)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
-		}
-		src = rootfs
-	}
-
-	toCopy := make(chan CopyJob)
-
-	var wg = new(sync.WaitGroup)
-	for i := 0; i < concurrency; i++ {
-		wg.Add(1)
-		go worker(i, wg, toCopy)
-	}
-
-	// Handle includes in spec. If specified they filter what gets in the package
-	if len(includes) > 0 {
-		var includeRegexp []*regexp.Regexp
-		for _, i := range includes {
-			r, e := regexp.Compile(i)
-			if e != nil {
-				Warning("Failed compiling regex:", e)
-				continue
-			}
-			includeRegexp = append(includeRegexp, r)
-		}
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-		ADDS:
-			for _, a := range l.Diffs.Additions {
-				for _, i := range includeRegexp {
-					if i.MatchString(a.Name) {
-						toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-						continue ADDS
-					}
-				}
-			}
-		}
-	} else {
-		// Otherwise just grab all
-		for _, l := range layers {
-			// Consider d.Additions (and d.Changes? - warn at least) only
-			for _, a := range l.Diffs.Additions {
-				toCopy <- CopyJob{Src: filepath.Join(src, a.Name), Dst: filepath.Join(archive, a.Name), Artifact: a.Name}
-			}
-		}
-	}
-
-	close(toCopy)
-	wg.Wait()
-
-	err = helpers.Tar(archive, dst)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error met while creating package archive")
-	}
-	return NewPackageArtifact(dst), nil
-}

pkg/compiler/artifact_test.go

@@ -1,140 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	. "github.com/mudler/luet/pkg/compiler/backend"
-
-	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Artifact", func() {
-	Context("Simple package build definition", func() {
-		It("Generates a delta", func() {
-
-			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
-
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
-
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
-
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
-			Expect(err).ToNot(HaveOccurred())
-
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
-			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
-			Expect(lspec.Image).To(Equal("luet/base"))
-			Expect(lspec.Seed).To(Equal("alpine"))
-			tmpdir, err := ioutil.TempDir(os.TempDir(), "tree")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(tmpdir) // clean up
-
-			tmpdir2, err := ioutil.TempDir(os.TempDir(), "tree2")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(tmpdir2) // clean up
-
-			unpacked, err := ioutil.TempDir(os.TempDir(), "unpacked")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(unpacked) // clean up
-
-			rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(rootfs) // clean up
-
-			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM alpine
-COPY . /luetbuild
-WORKDIR /luetbuild
-`))
-			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
-				ImageName:      "luet/base",
-				SourcePath:     tmpdir,
-				DockerFileName: "Dockerfile",
-				Destination:    filepath.Join(tmpdir2, "output1.tar"),
-			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
-			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
-
-			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM luet/base
-RUN echo foo > /test
-RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
-				ImageName:      "test",
-				SourcePath:     tmpdir,
-				DockerFileName: "LuetDockerfile",
-				Destination:    filepath.Join(tmpdir, "output2.tar"),
-			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
-			diffs, err := b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))
-			Expect(err).ToNot(HaveOccurred())
-
-			Expect(diffs).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
-					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
-					},
-				}}))
-			err = b.ExtractRootfs(CompilerBackendOptions{SourcePath: filepath.Join(tmpdir, "output2.tar"), Destination: rootfs}, false)
-			Expect(err).ToNot(HaveOccurred())
-
-			artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{})
-			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
-			err = helpers.Untar(artifact.GetPath(), unpacked, false)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
-			Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
-			content1, err := helpers.Read(filepath.Join(unpacked, "test"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content1).To(Equal("foo\n"))
-			content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(content2).To(Equal("bar\n"))
-		})
-
-	})
-})

pkg/compiler/backend.go

@@ -0,0 +1,245 @@
+package compiler
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
+
+	"github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/pkg/errors"
+)
+
+func NewBackend(ctx *types.Context, s string) (CompilerBackend, error) {
+	var compilerBackend CompilerBackend
+
+	switch s {
+	case backend.ImgBackend:
+		compilerBackend = backend.NewSimpleImgBackend(ctx)
+	case backend.DockerBackend:
+		compilerBackend = backend.NewSimpleDockerBackend(ctx)
+	default:
+		return nil, errors.New("invalid backend. Unsupported")
+	}
+
+	return compilerBackend, nil
+}
+
+type CompilerBackend interface {
+	BuildImage(backend.Options) error
+	ExportImage(backend.Options) error
+	RemoveImage(backend.Options) error
+	ImageDefinitionToTar(backend.Options) error
+	ExtractRootfs(opts backend.Options, keepPerms bool) error
+
+	CopyImage(string, string) error
+	DownloadImage(opts backend.Options) error
+
+	Push(opts backend.Options) error
+	ImageAvailable(string) bool
+
+	ImageExists(string) bool
+}
+
+// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
+// example of json return: [
+//   {
+//     "Image1": "luet/base",
+//     "Image2": "alpine",
+//     "DiffType": "File",
+//     "Diff": {
+//       "Adds": null,
+//       "Dels": [
+//         {
+//           "Name": "/luetbuild",
+//           "Size": 5830706
+//         },
+//         {
+//           "Name": "/luetbuild/Dockerfile",
+//           "Size": 50
+//         },
+//         {
+//           "Name": "/luetbuild/output1",
+//           "Size": 5830656
+//         }
+//       ],
+//       "Mods": null
+//     }
+//   }
+// ]
+func GenerateChanges(ctx *types.Context, b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
+
+	res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
+
+	tmpdiffs, err := ctx.Config.GetSystem().TempDir("extraction")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tmpdiffs) // clean up
+
+	srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(srcRootFS) // clean up
+
+	dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(dstRootFS) // clean up
+
+	srcImageExtract := backend.Options{
+		ImageName:   fromImage.ImageName,
+		Destination: srcRootFS,
+	}
+	ctx.Debug("Extracting source image", fromImage.ImageName)
+	err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
+	}
+
+	dstImageExtract := backend.Options{
+		ImageName:   toImage.ImageName,
+		Destination: dstRootFS,
+	}
+	ctx.Debug("Extracting destination image", toImage.ImageName)
+	err = b.ExtractRootfs(dstImageExtract, false)
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
+	}
+
+	// Get Additions/Changes. dst -> src
+	err = filepath.Walk(dstRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, dstRootFS, "", -1)
+		fileInfo, err := os.Lstat(filepath.Join(srcRootFS, realpath))
+		if err == nil {
+			var sizeA, sizeB int64
+			sizeA = fileInfo.Size()
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+
+			if sizeA != sizeB {
+				// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
+				res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
+					Name: filepath.Join("/", realpath),
+					Size: int(sizeB),
+				})
+			} else {
+				// fmt.Println("File already exists", path, filepath.Join(srcRootFS, realpath))
+			}
+		} else {
+			var sizeB int64
+
+			if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
+				sizeB = s.Size()
+			}
+			res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+				Size: int(sizeB),
+			})
+
+			// fmt.Println("File created", path, filepath.Join(srcRootFS, realpath))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
+	}
+
+	// Get deletions. src -> dst
+	err = filepath.Walk(srcRootFS, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+
+		realpath := strings.Replace(path, srcRootFS, "", -1)
+		if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
+			// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
+			res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
+				Name: filepath.Join("/", realpath),
+			})
+		}
+
+		return nil
+	})
+	if err != nil {
+		return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
+	}
+
+	diffs := []artifact.ArtifactLayer{res}
+
+	if ctx.Config.GetGeneral().Debug {
+		summary := ComputeArtifactLayerSummary(diffs)
+		for _, l := range summary.Layers {
+			ctx.Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
+				l.FromImage, l.ToImage,
+				l.AddFiles, l.AddSizes,
+				l.DelFiles, l.DelSizes,
+				l.ChangeFiles, l.ChangeSizes))
+		}
+	}
+
+	return diffs, nil
+}
+
+type ArtifactLayerSummary struct {
+	FromImage   string `json:"image1"`
+	ToImage     string `json:"image2"`
+	AddFiles    int    `json:"add_files"`
+	AddSizes    int64  `json:"add_sizes"`
+	DelFiles    int    `json:"del_files"`
+	DelSizes    int64  `json:"del_sizes"`
+	ChangeFiles int    `json:"change_files"`
+	ChangeSizes int64  `json:"change_sizes"`
+}
+
+type ArtifactLayersSummary struct {
+	Layers []ArtifactLayerSummary `json:"summary"`
+}
+
+func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
+
+	ans := ArtifactLayersSummary{
+		Layers: make([]ArtifactLayerSummary, 0),
+	}
+
+	for _, layer := range diffs {
+		sum := ArtifactLayerSummary{
+			FromImage:   layer.FromImage,
+			ToImage:     layer.ToImage,
+			AddFiles:    0,
+			AddSizes:    0,
+			DelFiles:    0,
+			DelSizes:    0,
+			ChangeFiles: 0,
+			ChangeSizes: 0,
+		}
+		for _, a := range layer.Diffs.Additions {
+			sum.AddFiles++
+			sum.AddSizes += int64(a.Size)
+		}
+		for _, d := range layer.Diffs.Deletions {
+			sum.DelFiles++
+			sum.DelSizes += int64(d.Size)
+		}
+		for _, c := range layer.Diffs.Changes {
+			sum.ChangeFiles++
+			sum.ChangeSizes += int64(c.Size)
+		}
+		ans.Layers = append(ans.Layers, sum)
+	}
+
+	return ans
+}

pkg/compiler/backend/common.go

@@ -0,0 +1,81 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"os/exec"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/pkg/errors"
+)
+
+const (
+	ImgBackend    = "img"
+	DockerBackend = "docker"
+)
+
+func imageAvailable(image string) bool {
+	_, err := crane.Digest(image)
+	return err == nil
+}
+
+type Options struct {
+	ImageName      string
+	SourcePath     string
+	DockerFileName string
+	Destination    string
+	Context        string
+	BackendArgs    []string
+}
+
+func runCommand(ctx *types.Context, cmd *exec.Cmd) error {
+	output := ""
+	buffered := !ctx.Config.GetGeneral().ShowBuildOutput
+	writer := NewBackendWriter(buffered, ctx)
+
+	cmd.Stdout = writer
+	cmd.Stderr = writer
+
+	if buffered {
+		ctx.Spinner()
+		defer ctx.SpinnerStop()
+	}
+
+	err := cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "Failed starting command")
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		output = writer.GetCombinedOutput()
+		return errors.Wrapf(err, "Failed running command: %s", output)
+	}
+
+	return nil
+}
+
+func genBuildCommand(opts Options) []string {
+	context := opts.Context
+
+	if context == "" {
+		context = "."
+	}
+	buildarg := append(opts.BackendArgs, "-f", opts.DockerFileName, "-t", opts.ImageName, context)
+	return append([]string{"build"}, buildarg...)
+}

pkg/compiler/backend/simpledocker.go

@@ -23,78 +23,128 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/mudler/luet/pkg/api/core/types"
+	bus "github.com/mudler/luet/pkg/bus"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+
 	capi "github.com/mudler/docker-companion/api"
 
-	"github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/helpers"
-	. "github.com/mudler/luet/pkg/logger"
 
 	"github.com/pkg/errors"
 )
 
-type SimpleDocker struct{}
+type SimpleDocker struct {
+	ctx *types.Context
+}
 
-func NewSimpleDockerBackend() compiler.CompilerBackend {
-	return &SimpleDocker{}
+func NewSimpleDockerBackend(ctx *types.Context) *SimpleDocker {
+	return &SimpleDocker{ctx: ctx}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
-	buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
-	Debug(":whale2: Building image " + name)
+	buildarg := genBuildCommand(opts)
+	s.ctx.Info(":whale2: Building image " + name)
 	cmd := exec.Command("docker", buildarg...)
-	cmd.Dir = path
-	out, err := cmd.CombinedOutput()
+	cmd.Dir = opts.SourcePath
+	err := runCommand(s.ctx, cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
-	Info(":whale: Building image " + name + " done")
 
-	//Info(string(out))
+	s.ctx.Success(":whale: Building image " + name + " done")
+
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
+
 	return nil
 }
 
-func (*SimpleDocker) CopyImage(src, dst string) error {
-	Debug(":whale: Tagging image:", src, "->", dst)
+func (s *SimpleDocker) CopyImage(src, dst string) error {
+	s.ctx.Debug(":whale: Tagging image:", src, "->", dst)
 	cmd := exec.Command("docker", "tag", src, dst)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed tagging image: "+string(out))
 	}
-	Info(":whale: Tagged image:", src, "->", dst)
+	s.ctx.Success(":whale: Tagged image:", src, "->", dst)
 	return nil
 }
 
-func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) DownloadImage(opts Options) error {
 	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
+
 	buildarg := []string{"pull", name}
-	Debug(":whale: Downloading image " + name)
+	s.ctx.Debug(":whale: Downloading image " + name)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
 	cmd := exec.Command("docker", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed pulling image: "+string(out))
 	}
-	Info(":whale: Downloaded image:", name)
+
+	s.ctx.Success(":whale: Downloaded image:", name)
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
+
 	return nil
 }
 
-func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) ImageExists(imagename string) bool {
+	buildarg := []string{"inspect", "--type=image", imagename}
+	s.ctx.Debug(":whale: Checking existance of docker image: " + imagename)
+	cmd := exec.Command("docker", buildarg...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		s.ctx.Debug("Image not present")
+		s.ctx.Debug(string(out))
+		return false
+	}
+	return true
+}
+
+func (*SimpleDocker) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+func (s *SimpleDocker) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rmi", name}
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
-	Info(":whale: Removed image:", name)
+	s.ctx.Success(":whale: Removed image:", name)
 	//Info(string(out))
 	return nil
 }
 
-func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) Push(opts Options) error {
+	name := opts.ImageName
+	pusharg := []string{"push", name}
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("docker", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	s.ctx.Success(":whale: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
+	//Info(string(out))
+	return nil
+}
+
+func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -107,18 +157,22 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions
 	return nil
 }
 
-func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleDocker) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
 
 	buildarg := []string{"save", name, "-o", path}
-	Debug(":whale: Saving image " + name)
+	s.ctx.Debug(":whale: Saving image " + name)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
 	out, err := exec.Command("docker", buildarg...).CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 
-	Info(":whale: Exported image:", name)
+	s.ctx.Debug(":whale: Exported image:", name)
 	return nil
 }
 
@@ -126,10 +180,46 @@ type ManifestEntry struct {
 	Layers []string `json:"Layers"`
 }
 
-func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
-	src := opts.SourcePath
+func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
+	name := opts.ImageName
 	dst := opts.Destination
 
+	if !b.ImageExists(name) {
+		if err := b.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
+	tempexport, err := ioutil.TempDir(dst, "tmprootfs")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+	}
+	defer os.RemoveAll(tempexport) // clean up
+
+	imageExport := filepath.Join(tempexport, "image.tar")
+
+	b.ctx.Spinner()
+	defer b.ctx.SpinnerStop()
+
+	if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
+		return errors.Wrap(err, "failed while extracting rootfs for "+name)
+	}
+
+	src := imageExport
+
+	if src == "" && opts.ImageName != "" {
+		tempUnpack, err := ioutil.TempDir(dst, "tempUnpack")
+		if err != nil {
+			return errors.Wrap(err, "Error met while creating tempdir for rootfs")
+		}
+		defer os.RemoveAll(tempUnpack) // clean up
+		imageExport := filepath.Join(tempUnpack, "image.tar")
+		if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
+			return errors.Wrap(err, "while exporting image before extraction")
+		}
+		src = imageExport
+	}
+
 	rootfs, err := ioutil.TempDir(dst, "tmprootfs")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for rootfs")
@@ -148,7 +238,7 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 		return errors.Wrap(err, "Error met while unpacking rootfs")
 	}
 
-	manifest, err := helpers.Read(filepath.Join(rootfs, "manifest.json"))
+	manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
 	if err != nil {
 		return errors.Wrap(err, "Error met while reading image manifest")
 	}
@@ -162,19 +252,21 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 
 	layers_sha := []string{}
 
-	if len(manifestData) != 1 {
-		return errors.New("Manifest should have one entry")
-	}
-	for _, l := range manifestData[0].Layers {
-		layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
-	}
+	for _, data := range manifestData {
 
+		for _, l := range data.Layers {
+			if strings.Contains(l, "layer.tar") {
+				layers_sha = append(layers_sha, strings.Replace(l, "/layer.tar", "", -1))
+			}
+		}
+	}
+	// TODO: Drop capi in favor of the img approach already used in pkg/installer/repository
 	export, err := capi.CreateExport(rootfs)
 	if err != nil {
 		return err
 	}
 
-	err = export.UnPackLayers(layers_sha, dst, "")
+	err = export.UnPackLayers(layers_sha, dst, "containerd")
 	if err != nil {
 		return err
 	}
@@ -186,52 +278,3 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 
 	return nil
 }
-
-// 	container-diff diff daemon://luet/base alpine --type=file -j
-// [
-//   {
-//     "Image1": "luet/base",
-//     "Image2": "alpine",
-//     "DiffType": "File",
-//     "Diff": {
-//       "Adds": null,
-//       "Dels": [
-//         {
-//           "Name": "/luetbuild",
-//           "Size": 5830706
-//         },
-//         {
-//           "Name": "/luetbuild/Dockerfile",
-//           "Size": 50
-//         },
-//         {
-//           "Name": "/luetbuild/output1",
-//           "Size": 5830656
-//         }
-//       ],
-//       "Mods": null
-//     }
-//   }
-// ]
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := ioutil.TempDir(os.TempDir(), "tmpdiffs")
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
-	}
-	defer os.RemoveAll(tmpdiffs) // clean up
-
-	diffargs := []string{"diff", fromImage, toImage, "--type=file", "-j", "-n", "-c", tmpdiffs}
-	out, err := exec.Command("container-diff", diffargs...).CombinedOutput()
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed Resolving layer diffs: "+string(out))
-	}
-
-	var diffs []compiler.ArtifactLayer
-
-	err = json.Unmarshal(out, &diffs)
-	if err != nil {
-		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Failed unmarshalling json response: "+string(out))
-	}
-	return diffs, nil
-}

pkg/compiler/backend/simpledocker_test.go

@@ -16,14 +16,18 @@
 package backend_test
 
 import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/api/core/types/artifact"
+	"github.com/mudler/luet/pkg/compiler"
 	. "github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/compiler/backend"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	"github.com/mudler/luet/pkg/tree"
 	. "github.com/onsi/ginkgo"
@@ -32,22 +36,19 @@ import (
 
 var _ = Describe("Docker backend", func() {
 	Context("Simple Docker backend satisfies main interface functionalities", func() {
+		ctx := types.NewContext()
 		It("Builds and generate tars", func() {
 			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
 
 			err := generalRecipe.Load("../../../tests/fixtures/buildtree")
 			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
 
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
 
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
+			cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
+			lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
 			Expect(err).ToNot(HaveOccurred())
 
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
 			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 			Expect(lspec.Image).To(Equal("luet/base"))
 			Expect(lspec.Seed).To(Equal("alpine"))
@@ -61,54 +62,87 @@ var _ = Describe("Docker backend", func() {
 
 			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
+			dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM alpine
 COPY . /luetbuild
 WORKDIR /luetbuild
-`))
-			b := NewSimpleDockerBackend()
-			opts := CompilerBackendOptions{
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin`))
+			b := NewSimpleDockerBackend(ctx)
+			opts := backend.Options{
 				ImageName:      "luet/base",
 				SourcePath:     tmpdir,
 				DockerFileName: "Dockerfile",
 				Destination:    filepath.Join(tmpdir2, "output1.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
+
 			Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
 
 			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
+			dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
+WORKDIR /luetbuild
+ENV PACKAGE_NAME=enman
+ENV PACKAGE_VERSION=1.4.0
+ENV PACKAGE_CATEGORY=app-admin
 RUN echo foo > /test
 RUN echo bar > /test2`))
-			opts = CompilerBackendOptions{
+			opts2 := backend.Options{
 				ImageName:      "test",
 				SourcePath:     tmpdir,
 				DockerFileName: "LuetDockerfile",
 				Destination:    filepath.Join(tmpdir, "output2.tar"),
 			}
-			Expect(b.ImageDefinitionToTar(opts)).ToNot(HaveOccurred())
-			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			Expect(b.Changes(filepath.Join(tmpdir2, "output1.tar"), filepath.Join(tmpdir, "output2.tar"))).To(Equal(
-				[]ArtifactLayer{{
-					FromImage: filepath.Join(tmpdir2, "output1.tar"),
-					ToImage:   filepath.Join(tmpdir, "output2.tar"),
-					Diffs: ArtifactDiffs{
-						Additions: []ArtifactNode{
-							{Name: "/test", Size: 4},
-							{Name: "/test2", Size: 4},
-						},
+			Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
+			Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
+
+			artifacts := []artifact.ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
+			if os.Getenv("DOCKER_BUILDKIT") == "1" {
+				artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
+			}
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
+			artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
+
+			Expect(compiler.GenerateChanges(ctx, b, opts, opts2)).To(Equal(
+				[]artifact.ArtifactLayer{{
+					FromImage: "luet/base",
+					ToImage:   "test",
+					Diffs: artifact.ArtifactDiffs{
+						Additions: artifacts,
 					},
 				}}))
 
+			opts2 = backend.Options{
+				ImageName:      "test",
+				SourcePath:     tmpdir,
+				DockerFileName: "LuetDockerfile",
+				Destination:    filepath.Join(tmpdir, "output3.tar"),
+			}
+
+			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
+			Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
+			Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
 		})
 
+		It("Detects available images", func() {
+			b := NewSimpleDockerBackend(ctx)
+			Expect(b.ImageAvailable("quay.io/mocaccino/extra")).To(BeTrue())
+			Expect(b.ImageAvailable("ubuntu:20.10")).To(BeTrue())
+			Expect(b.ImageAvailable("igjo5ijgo25nho52")).To(BeFalse())
+		})
 	})
 })

pkg/compiler/backend/simpleimg.go

@@ -16,78 +16,114 @@
 package backend
 
 import (
+	"os"
 	"os/exec"
+	"strings"
 
-	"github.com/mudler/luet/pkg/compiler"
-	. "github.com/mudler/luet/pkg/logger"
+	"github.com/mudler/luet/pkg/api/core/types"
+	bus "github.com/mudler/luet/pkg/bus"
 
 	"github.com/pkg/errors"
 )
 
-type SimpleImg struct{}
+type SimpleImg struct {
+	ctx *types.Context
+}
 
-func NewSimpleImgBackend() compiler.CompilerBackend {
-	return &SimpleImg{}
+func NewSimpleImgBackend(ctx *types.Context) *SimpleImg {
+	return &SimpleImg{ctx: ctx}
 }
 
 // TODO: Missing still: labels, and build args expansion
-func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) BuildImage(opts Options) error {
 	name := opts.ImageName
-	path := opts.SourcePath
-	dockerfileName := opts.DockerFileName
+	bus.Manager.Publish(bus.EventImagePreBuild, opts)
 
-	buildarg := []string{"build", "-t", name, path, "-f ", dockerfileName}
-	Spinner(22)
-	Debug("Building image "+name+" - running img with: ", buildarg)
-	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	buildarg := genBuildCommand(opts)
+
+	s.ctx.Info(":tea: Building image " + name)
+
+	cmd := exec.Command("img", buildarg...)
+	cmd.Dir = opts.SourcePath
+	err := runCommand(s.ctx, cmd)
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return err
 	}
-	SpinnerStop()
-	Info(out)
+	bus.Manager.Publish(bus.EventImagePostBuild, opts)
+
+	s.ctx.Info(":tea: Building image " + name + " done")
+
 	return nil
 }
 
-func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) RemoveImage(opts Options) error {
 	name := opts.ImageName
 	buildarg := []string{"rm", name}
-	Spinner(22)
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
-	SpinnerStop()
-	Info(out)
+
+	s.ctx.Info(":tea: Image " + name + " removed")
 	return nil
 }
 
-func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
-
+func (s *SimpleImg) DownloadImage(opts Options) error {
 	name := opts.ImageName
-	buildarg := []string{"pull", name}
+	bus.Manager.Publish(bus.EventImagePrePull, opts)
+
+	buildarg := []string{"pull", name}
+	s.ctx.Debug(":tea: Downloading image " + name)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
-	Debug("Downloading image "+name+" - running img with: ", buildarg)
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
+
+	s.ctx.Info(":tea: Image " + name + " downloaded")
+	bus.Manager.Publish(bus.EventImagePostPull, opts)
+
 	return nil
 }
-func (*SimpleImg) CopyImage(src, dst string) error {
-	Spinner(22)
-	defer SpinnerStop()
+func (s *SimpleImg) CopyImage(src, dst string) error {
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
 
-	Debug("Tagging image - running img with: ", src, dst)
+	s.ctx.Debug(":tea: Tagging image", src, dst)
 	cmd := exec.Command("img", "tag", src, dst)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return errors.Wrap(err, "Failed tagging image: "+string(out))
 	}
+	s.ctx.Info(":tea: Image " + dst + " tagged")
+
 	return nil
 }
 
-func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) ImageAvailable(imagename string) bool {
+	return imageAvailable(imagename)
+}
+
+// ImageExists check if the given image is available locally
+func (*SimpleImg) ImageExists(imagename string) bool {
+	cmd := exec.Command("img", "ls")
+	out, err := cmd.Output()
+	if err != nil {
+		return false
+	}
+	if strings.Contains(string(out), imagename) {
+		return true
+	}
+	return false
+}
+
+func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
 	if err := s.BuildImage(opts); err != nil {
 		return errors.Wrap(err, "Failed building image")
 	}
@@ -100,25 +136,62 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
 	return nil
 }
 
-func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
+func (s *SimpleImg) ExportImage(opts Options) error {
 	name := opts.ImageName
 	path := opts.Destination
-	buildarg := []string{"save", name, "-o", path}
-	Debug("Saving image "+name+" - running img with: ", buildarg)
+	buildarg := []string{"save", "-o", path, name}
+	s.ctx.Debug(":tea: Saving image " + name)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
+	s.ctx.Info(":tea: Image " + name + " saved")
 	return nil
 }
 
-// TODO: Dup in docker, refactor common code in helpers for shared parts
-func (*SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
-	return NewSimpleDockerBackend().ExtractRootfs(opts, keepPerms)
+// ExtractRootfs extracts the docker image content inside the destination
+func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
+	name := opts.ImageName
+	path := opts.Destination
+
+	if !s.ImageExists(name) {
+		if err := s.DownloadImage(opts); err != nil {
+			return errors.Wrap(err, "failed pulling image "+name+" during extraction")
+		}
+	}
+
+	os.RemoveAll(path)
+
+	buildarg := []string{"unpack", "-o", path, name}
+	s.ctx.Debug(":tea: Extracting image " + name)
+
+	s.ctx.Spinner()
+	defer s.ctx.SpinnerStop()
+
+	out, err := exec.Command("img", buildarg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed extracting image: "+string(out))
+	}
+	s.ctx.Debug(":tea: Image " + name + " extracted")
+	return nil
 }
 
-// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
-// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
-func (*SimpleImg) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	return NewSimpleDockerBackend().Changes(fromImage, toImage)
+func (s *SimpleImg) Push(opts Options) error {
+	name := opts.ImageName
+	bus.Manager.Publish(bus.EventImagePrePush, opts)
+
+	pusharg := []string{"push", name}
+	out, err := exec.Command("img", pusharg...).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, "Failed pushing image: "+string(out))
+	}
+	s.ctx.Info(":tea: Pushed image:", name)
+	bus.Manager.Publish(bus.EventImagePostPush, opts)
+
+	//s.ctx.Info(string(out))
+	return nil
 }

pkg/compiler/backend/writer.go

@@ -0,0 +1,50 @@
+// Copyright © 2021 Daniele Rondina <geaaru@sabayonlinux.org>
+//                  Ettore Di Giacinto <mudler@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"bytes"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+)
+
+type BackendWriter struct {
+	BufferedOutput bool
+	Buffer         *bytes.Buffer
+	ctx            *types.Context
+}
+
+func NewBackendWriter(buffered bool, ctx *types.Context) *BackendWriter {
+	return &BackendWriter{
+		BufferedOutput: buffered,
+		Buffer:         &bytes.Buffer{},
+		ctx:            ctx,
+	}
+}
+
+func (b *BackendWriter) Write(p []byte) (int, error) {
+	if b.BufferedOutput {
+		return b.Buffer.Write(p)
+	}
+
+	b.ctx.Msg("info", false, (string(p)))
+
+	return len(p), nil
+}
+
+func (b *BackendWriter) Close() error              { return nil }
+func (b *BackendWriter) GetCombinedOutput() string { return b.Buffer.String() }

pkg/compiler/backend_test.go

@@ -0,0 +1,79 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/compiler"
+	. "github.com/mudler/luet/pkg/compiler/backend"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Docker image diffs", func() {
+	var b CompilerBackend
+	ctx := types.NewContext()
+	BeforeEach(func() {
+		b = NewSimpleDockerBackend(ctx)
+	})
+
+	Context("Generate diffs from docker images", func() {
+		It("Detect no changes", func() {
+			opts := Options{
+				ImageName: "alpine:latest",
+			}
+			err := b.DownloadImage(opts)
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(ctx, b, opts, opts)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+			Expect(len(layers[0].Diffs.Additions)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Changes)).To(Equal(0))
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+
+		It("Detects additions and changed files", func() {
+			err := b.DownloadImage(Options{
+				ImageName: "quay.io/mocaccino/micro",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			err = b.DownloadImage(Options{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+
+			layers, err := GenerateChanges(ctx, b, Options{
+				ImageName: "quay.io/mocaccino/micro",
+			}, Options{
+				ImageName: "quay.io/mocaccino/extra",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(layers)).To(Equal(1))
+
+			Expect(len(layers[0].Diffs.Changes) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Changes[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Changes[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Additions) > 0).To(BeTrue())
+			Expect(len(layers[0].Diffs.Additions[0].Name) > 0).To(BeTrue())
+			Expect(layers[0].Diffs.Additions[0].Size > 0).To(BeTrue())
+
+			Expect(len(layers[0].Diffs.Deletions)).To(Equal(0))
+		})
+	})
+})

pkg/compiler/buildtree.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+func rankMapStringInt(values map[string]int) []string {
+	type kv struct {
+		Key   string
+		Value int
+	}
+	var ss []kv
+	for k, v := range values {
+		ss = append(ss, kv{k, v})
+	}
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Value > ss[j].Value
+	})
+	ranked := make([]string, len(values))
+	for i, kv := range ss {
+		ranked[i] = kv.Key
+	}
+	return ranked
+}
+
+type BuildTree struct {
+	order map[string]int
+}
+
+func (bt *BuildTree) Increase(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	if _, ok := bt.order[s]; !ok {
+		bt.order[s] = 0
+	}
+
+	bt.order[s]++
+}
+
+func (bt *BuildTree) Reset(s string) {
+	if bt.order == nil {
+		bt.order = make(map[string]int)
+	}
+	bt.order[s] = 0
+}
+
+func (bt *BuildTree) Level(s string) int {
+	return bt.order[s]
+}
+
+func ints(input []int) []int {
+	u := make([]int, 0, len(input))
+	m := make(map[int]bool)
+
+	for _, val := range input {
+		if _, ok := m[val]; !ok {
+			m[val] = true
+			u = append(u, val)
+		}
+	}
+
+	return u
+}
+
+func (bt *BuildTree) AllInLevel(l int) []string {
+	var all []string
+	for k, v := range bt.order {
+		if v == l {
+			all = append(all, k)
+		}
+	}
+	return all
+}
+
+func (bt *BuildTree) Order(compilationTree map[string]map[string]interface{}) {
+	sentinel := false
+	for !sentinel {
+		sentinel = true
+
+	LEVEL:
+		for _, l := range bt.AllLevels() {
+
+			for _, j := range bt.AllInLevel(l) {
+				for _, k := range bt.AllInLevel(l) {
+					if j == k {
+						continue
+					}
+					if _, ok := compilationTree[j][k]; ok {
+						if bt.Level(j) == bt.Level(k) {
+							bt.Increase(j)
+							sentinel = false
+							break LEVEL
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func (bt *BuildTree) AllLevels() []int {
+	var all []int
+	for _, v := range bt.order {
+		all = append(all, v)
+	}
+
+	sort.Sort(sort.IntSlice(all))
+
+	return ints(all)
+}
+
+func (bt *BuildTree) JSON() (string, error) {
+	type buildjob struct {
+		Jobs []string `json:"packages"`
+	}
+
+	result := []buildjob{}
+	for _, l := range bt.AllLevels() {
+		result = append(result, buildjob{Jobs: bt.AllInLevel(l)})
+	}
+	dat, err := json.Marshal(&result)
+	return string(dat), err
+}

pkg/compiler/imagehashtree.go

@@ -0,0 +1,161 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"fmt"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+	"github.com/pkg/errors"
+)
+
+// ImageHashTree is holding the Database
+// and the options to resolve PackageImageHashTrees
+// for a given specfile
+// It is responsible of returning a concrete result
+// which identifies a Package in a HashTree
+type ImageHashTree struct {
+	Database      pkg.PackageDatabase
+	SolverOptions types.LuetSolverOptions
+}
+
+// PackageImageHashTree represent the Package into a given image hash tree
+// The hash tree is constructed by a set of images representing
+// the package during its build stage. A Hash is assigned to each image
+// from the package fingerprint, plus the SAT solver assertion result (which is hashed as well)
+// and the specfile signatures. This guarantees that each image of the build stage
+// is unique and can be identified later on.
+type PackageImageHashTree struct {
+	Target                       *solver.PackageAssert
+	Dependencies                 solver.PackagesAssertions
+	Solution                     solver.PackagesAssertions
+	dependencyBuilderImageHashes map[string]string
+	SourceHash                   string
+	BuilderImageHash             string
+}
+
+func NewHashTree(db pkg.PackageDatabase) *ImageHashTree {
+	return &ImageHashTree{
+		Database: db,
+	}
+}
+
+func (ht *PackageImageHashTree) DependencyBuildImage(p pkg.Package) (string, error) {
+	found, ok := ht.dependencyBuilderImageHashes[p.GetFingerPrint()]
+	if !ok {
+		return "", errors.New("package hash not found")
+	}
+	return found, nil
+}
+
+func (ht *PackageImageHashTree) String() string {
+	return fmt.Sprintf(
+		"Target buildhash: %s\nTarget packagehash: %s\nBuilder Imagehash: %s\nSource Imagehash: %s\n",
+		ht.Target.Hash.BuildHash,
+		ht.Target.Hash.PackageHash,
+		ht.BuilderImageHash,
+		ht.SourceHash,
+	)
+}
+
+// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
+// PackageImageHashTree contains all the informations to resolve the spec build images in order to
+// reproducibly re-build images from packages
+func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
+	assertions, err := ht.resolve(cs, p)
+	if err != nil {
+		return nil, err
+	}
+	targetAssertion := assertions.Search(p.GetPackage().GetFingerPrint())
+
+	dependencies := assertions.Drop(p.GetPackage())
+	var sourceHash string
+	imageHashes := map[string]string{}
+	for _, assertion := range dependencies {
+		var depbuildImageTag string
+		compileSpec, err := cs.FromPackage(assertion.Package)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error while generating compilespec for "+assertion.Package.GetName())
+		}
+		if compileSpec.GetImage() != "" {
+			depbuildImageTag = assertion.Hash.BuildHash
+		} else {
+			depbuildImageTag = ht.genBuilderImageTag(compileSpec, targetAssertion.Hash.PackageHash)
+		}
+		imageHashes[assertion.Package.GetFingerPrint()] = depbuildImageTag
+		sourceHash = assertion.Hash.PackageHash
+	}
+
+	return &PackageImageHashTree{
+		Dependencies:                 dependencies,
+		Target:                       targetAssertion,
+		SourceHash:                   sourceHash,
+		BuilderImageHash:             ht.genBuilderImageTag(p, targetAssertion.Hash.PackageHash),
+		dependencyBuilderImageHashes: imageHashes,
+		Solution:                     assertions,
+	}, nil
+}
+
+func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
+	// Use packageImage as salt into the fp being used
+	// so the hash is unique also in cases where
+	// some package deps does have completely different
+	// depgraphs
+	return fmt.Sprintf("builder-%s", p.GetPackage().HashFingerprint(packageImage))
+}
+
+// resolve computes the dependency tree of a compilation spec and returns solver assertions
+// in order to be able to compile the spec.
+func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
+	dependencies, err := cs.ComputeDepTree(p)
+	if err != nil {
+		return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())
+	}
+
+	// Get hash from buildpsecs
+	salts := map[string]string{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			spec, err := cs.FromPackage(assertion.Package)
+			if err != nil {
+				return nil, errors.Wrap(err, "while computing hash buildspecs")
+			}
+			hash, err := spec.Hash()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed computing hash")
+			}
+			salts[assertion.Package.GetFingerPrint()] = hash
+		}
+	}
+
+	assertions := solver.PackagesAssertions{}
+	for _, assertion := range dependencies { //highly dependent on the order
+		if assertion.Value {
+			nthsolution := dependencies.Cut(assertion.Package)
+			assertion.Hash = solver.PackageHash{
+				BuildHash:   nthsolution.SaltedHashFrom(assertion.Package, salts),
+				PackageHash: nthsolution.SaltedAssertionHash(salts),
+			}
+			assertion.Package.SetTreeDir(p.Package.GetTreeDir())
+			assertions = append(assertions, assertion)
+		}
+	}
+
+	return assertions, nil
+}

pkg/compiler/imagehashtree_test.go

@@ -0,0 +1,151 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package compiler_test
+
+import (
+	"github.com/mudler/luet/pkg/api/core/types"
+	. "github.com/mudler/luet/pkg/compiler"
+	sd "github.com/mudler/luet/pkg/compiler/backend"
+	"github.com/mudler/luet/pkg/compiler/types/options"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/tree"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ImageHashTree", func() {
+	ctx := types.NewContext()
+	generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+	hashtree := NewHashTree(generalRecipe.GetDatabase())
+	Context("Simple package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			err := generalRecipe.Load("../../tests/fixtures/buildable")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+
+		It("Calculates the hash correctly", func() {
+
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Target.Hash.BuildHash).To(Equal("895697a8bb51b219b78ed081fa1b778801e81505bb03f56acafcf3c476620fc1"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("2a6c3dc0dd7af2902fd8823a24402d89b2030cfbea6e63fe81afb34af8b1a005"))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-3a28d240f505d69123735a567beaf80e"))
+		})
+	})
+
+	expectedPackageHash := "f3f42a7435293225e92a51da8416f90b7c0ccd5958cd5c72276c39ece408c01f"
+
+	Context("complex package definition", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-977129605c0d7e974cc8a431a563cec1"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("9112e2c97bf8ca998c1df303a9ebc4957b685930c882e9aa556eab4507220079"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+			Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
+		})
+	})
+
+	Context("complex package definition, with small change in build.yaml", func() {
+		BeforeEach(func() {
+			generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+
+			//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
+			err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
+			Expect(err).ToNot(HaveOccurred())
+			compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
+			hashtree = NewHashTree(generalRecipe.GetDatabase())
+
+		})
+		It("Calculates the hash correctly", func() {
+			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			packageHash, err := hashtree.Query(compiler, spec)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+			sourceHash := "f0d96e79a0021b065674c58203158a02ed702ff29cc2ee6605e1f16f1fae9b89"
+			Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
+			Expect(packageHash.SourceHash).To(Equal(sourceHash))
+
+			Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
+
+			Expect(packageHash.BuilderImageHash).To(Equal("builder-05506adb3777ed74825254e8b46da5e9"))
+
+			//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
+			Expect(packageHash.Target.Hash.PackageHash).To(Equal("489ece9825e18eba2de91e93293d5c61578d492d42bbfe9b6ae2e2c3c11bc842"))
+			a := &pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.1"}
+			hash, err := packageHash.DependencyBuildImage(a)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+
+			assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
+
+			Expect(assertionA.Hash.PackageHash).To(Equal("f0d96e79a0021b065674c58203158a02ed702ff29cc2ee6605e1f16f1fae9b89"))
+			Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
+
+			b := &pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}
+			assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
+
+			Expect(assertionB.Hash.PackageHash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
+			hashB, err := packageHash.DependencyBuildImage(b)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
+		})
+	})
+
+})

pkg/compiler/interface.go

@@ -1,118 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-)
-
-type Compiler interface {
-	Compile(int, bool, CompilationSpec) (Artifact, error)
-	CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
-	ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
-	Prepare(concurrency int) error
-
-	FromPackage(pkg.Package) (CompilationSpec, error)
-
-	SetBackend(CompilerBackend)
-	GetBackend() CompilerBackend
-}
-
-type CompilerBackendOptions struct {
-	ImageName      string
-	SourcePath     string
-	DockerFileName string
-	Destination    string
-}
-
-type CompilerBackend interface {
-	BuildImage(CompilerBackendOptions) error
-	ExportImage(CompilerBackendOptions) error
-	RemoveImage(CompilerBackendOptions) error
-	Changes(fromImage, toImage string) ([]ArtifactLayer, error)
-	ImageDefinitionToTar(CompilerBackendOptions) error
-	ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
-
-	CopyImage(string, string) error
-	DownloadImage(opts CompilerBackendOptions) error
-}
-
-type Artifact interface {
-	GetPath() string
-	SetPath(string)
-	GetDependencies() []Artifact
-	SetDependencies(d []Artifact)
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-
-	SetCompileSpec(as CompilationSpec)
-	GetCompileSpec() CompilationSpec
-}
-
-type ArtifactNode struct {
-	Name string `json:"Name"`
-	Size int    `json:"Size"`
-}
-type ArtifactDiffs struct {
-	Additions []ArtifactNode `json:"Adds"`
-	Deletions []ArtifactNode `json:"Dels"`
-	Changes   []ArtifactNode `json:"Mods"`
-}
-type ArtifactLayer struct {
-	FromImage string        `json:"Image1"`
-	ToImage   string        `json:"Image2"`
-	Diffs     ArtifactDiffs `json:"Diff"`
-}
-
-// CompilationSpec represent a compilation specification derived from a package
-type CompilationSpec interface {
-	ImageUnpack() bool // tells if the definition is just an image
-	GetIncludes() []string
-
-	RenderBuildImage() (string, error)
-	WriteBuildImageDefinition(string) error
-
-	RenderStepImage(image string) (string, error)
-	WriteStepImageDefinition(fromimage, path string) error
-
-	GetPackage() pkg.Package
-	BuildSteps() []string
-
-	GetSeedImage() string
-	SetSeedImage(string)
-
-	GetImage() string
-	SetImage(string)
-
-	SetOutputPath(string)
-	GetOutputPath() string
-	Rel(string) string
-
-	GetPreBuildSteps() []string
-
-	GetSourceAssertion() solver.PackagesAssertions
-	SetSourceAssertion(as solver.PackagesAssertions)
-}
-
-type CompilationSpecs interface {
-	Unique() CompilationSpecs
-	Len() int
-	All() []CompilationSpec
-	Add(CompilationSpec)
-	Remove(s CompilationSpecs) CompilationSpecs
-}

pkg/compiler/spec.go

@@ -1,206 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler
-
-import (
-	"io/ioutil"
-	"path/filepath"
-
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/solver"
-	yaml "gopkg.in/yaml.v2"
-)
-
-type LuetCompilationspecs []LuetCompilationSpec
-
-func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
-	all := LuetCompilationspecs{}
-
-	for _, spec := range s {
-		all.Add(spec)
-	}
-	return &all
-}
-
-func (specs LuetCompilationspecs) Len() int {
-	return len(specs)
-}
-
-func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
-	newSpecs := LuetCompilationspecs{}
-SPECS:
-	for _, spec := range specs.All() {
-		for _, target := range s.All() {
-			if target.GetPackage().Matches(spec.GetPackage()) {
-				continue SPECS
-			}
-		}
-		newSpecs.Add(spec)
-	}
-	return &newSpecs
-}
-
-func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
-	c, ok := s.(*LuetCompilationSpec)
-	if !ok {
-		panic("LuetCompilationspecs supports only []LuetCompilationSpec")
-	}
-	*specs = append(*specs, *c)
-}
-
-func (specs *LuetCompilationspecs) All() []CompilationSpec {
-	var cspecs []CompilationSpec
-	for i, _ := range *specs {
-		f := (*specs)[i]
-		cspecs = append(cspecs, &f)
-	}
-
-	return cspecs
-}
-
-func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
-	newSpecs := LuetCompilationspecs{}
-	seen := map[string]bool{}
-
-	for i, _ := range *specs {
-		j := (*specs)[i]
-		_, ok := seen[j.GetPackage().GetFingerPrint()]
-		if !ok {
-			seen[j.GetPackage().GetFingerPrint()] = true
-			newSpecs = append(newSpecs, j)
-		}
-	}
-	return &newSpecs
-}
-
-type LuetCompilationSpec struct {
-	Steps           []string                  `json:"steps"`   // Are run inside a container and the result layer diff is saved
-	Prelude         []string                  `json:"prelude"` // Are run inside the image which will be our builder
-	Image           string                    `json:"image"`
-	Seed            string                    `json:"seed"`
-	Package         pkg.Package               `json:"-"`
-	SourceAssertion solver.PackagesAssertions `json:"-"`
-
-	OutputPath string   `json:"-"` // Where the build processfiles go
-	Unpack     bool     `json:"unpack"`
-	Includes   []string `json:"includes"`
-}
-
-func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
-	var spec LuetCompilationSpec
-	err := yaml.Unmarshal(b, &spec)
-	if err != nil {
-		return &spec, err
-	}
-	spec.Package = p
-	return &spec, nil
-}
-func (a *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
-	return a.SourceAssertion
-}
-
-func (a *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
-	a.SourceAssertion = as
-}
-func (cs *LuetCompilationSpec) GetPackage() pkg.Package {
-	return cs.Package
-}
-
-func (cs *LuetCompilationSpec) BuildSteps() []string {
-	return cs.Steps
-}
-
-func (cs *LuetCompilationSpec) ImageUnpack() bool {
-	return cs.Unpack
-}
-
-func (cs *LuetCompilationSpec) GetPreBuildSteps() []string {
-	return cs.Prelude
-}
-
-func (cs *LuetCompilationSpec) GetIncludes() []string {
-	return cs.Includes
-}
-
-func (cs *LuetCompilationSpec) GetSeedImage() string {
-	return cs.Seed
-}
-
-func (cs *LuetCompilationSpec) GetImage() string {
-	return cs.Image
-}
-
-func (cs *LuetCompilationSpec) GetOutputPath() string {
-	return cs.OutputPath
-}
-
-func (p *LuetCompilationSpec) Rel(s string) string {
-	return filepath.Join(p.GetOutputPath(), s)
-}
-
-func (cs *LuetCompilationSpec) SetImage(s string) {
-	cs.Image = s
-}
-
-func (cs *LuetCompilationSpec) SetOutputPath(s string) {
-	cs.OutputPath = s
-}
-
-func (cs *LuetCompilationSpec) SetSeedImage(s string) {
-	cs.Seed = s
-}
-
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
-	spec := `
-FROM ` + cs.GetSeedImage() + `
-COPY . /luetbuild
-WORKDIR /luetbuild
-`
-	for _, s := range cs.GetPreBuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-	return spec, nil
-}
-
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
-	spec := `
-FROM ` + image
-	for _, s := range cs.BuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-
-	return spec, nil
-}
-
-func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {
-	data, err := cs.RenderBuildImage()
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(path, []byte(data), 0644)
-}
-
-func (cs *LuetCompilationSpec) WriteStepImageDefinition(fromimage, path string) error {
-	data, err := cs.RenderStepImage(fromimage)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(path, []byte(data), 0644)
-}

pkg/compiler/spec_test.go

@@ -1,101 +0,0 @@
-// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package compiler_test
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	. "github.com/mudler/luet/pkg/compiler"
-	helpers "github.com/mudler/luet/pkg/helpers"
-	pkg "github.com/mudler/luet/pkg/package"
-	"github.com/mudler/luet/pkg/tree"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Spec", func() {
-	Context("Luet specs", func() {
-		It("Allows normal operations", func() {
-			testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
-			testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
-			testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
-			testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
-
-			specs := NewLuetCompilationspecs(testSpec, testSpec2)
-			Expect(specs.Len()).To(Equal(2))
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
-			specs.Add(testSpec3)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
-			specs.Add(testSpec4)
-			Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
-			newSpec := specs.Unique()
-			Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
-
-			newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
-			Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
-
-		})
-	})
-
-	Context("Simple package build definition", func() {
-		It("Loads it correctly", func() {
-			generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
-
-			err := generalRecipe.Load("../../tests/fixtures/buildtree")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(generalRecipe.Tree()).ToNot(BeNil()) // It should be populated back at this point
-
-			Expect(len(generalRecipe.Tree().GetPackageSet().GetPackages())).To(Equal(1))
-
-			compiler := NewLuetCompiler(nil, generalRecipe.Tree(), generalRecipe.Tree().GetPackageSet())
-			spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
-			Expect(err).ToNot(HaveOccurred())
-
-			lspec, ok := spec.(*LuetCompilationSpec)
-			Expect(ok).To(BeTrue())
-
-			Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
-			Expect(lspec.Image).To(Equal("luet/base"))
-			Expect(lspec.Seed).To(Equal("alpine"))
-			tmpdir, err := ioutil.TempDir("", "tree")
-			Expect(err).ToNot(HaveOccurred())
-			defer os.RemoveAll(tmpdir) // clean up
-
-			err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM alpine
-COPY . /luetbuild
-WORKDIR /luetbuild
-`))
-
-			err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(dockerfile).To(Equal(`
-FROM luet/base
-RUN echo foo > /test
-RUN echo bar > /test2`))
-
-		})
-
-	})
-})

pkg/compiler/types/compression/compression.go

@@ -0,0 +1,9 @@
+package compression
+
+type Implementation string
+
+const (
+	None      Implementation = "none" // e.g. tar for standard packages
+	GZip      Implementation = "gzip"
+	Zstandard Implementation = "zstd"
+)

pkg/compiler/types/options/compiler_options.go

@@ -0,0 +1,218 @@
+// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package options
+
+import (
+	"runtime"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	"github.com/mudler/luet/pkg/compiler/types/compression"
+	"github.com/mudler/luet/pkg/solver"
+)
+
+type Compiler struct {
+	PushImageRepository      string
+	PullImageRepository      []string
+	PullFirst, KeepImg, Push bool
+	Concurrency              int
+	CompressionType          compression.Implementation
+
+	Wait            bool
+	OnlyDeps        bool
+	NoDeps          bool
+	SolverOptions   types.LuetSolverOptions
+	BuildValuesFile []string
+	BuildValues     []map[string]interface{}
+
+	PackageTargetOnly bool
+	Rebuild           bool
+
+	BackendArgs []string
+
+	BackendType string
+
+	// TemplatesFolder. should default to tree/templates
+	TemplatesFolder []string
+
+	Context *types.Context
+}
+
+func NewDefaultCompiler() *Compiler {
+	return &Compiler{
+		PushImageRepository: "luet/cache",
+		PullFirst:           false,
+		Push:                false,
+		CompressionType:     compression.None,
+		KeepImg:             true,
+		Concurrency:         runtime.NumCPU(),
+		OnlyDeps:            false,
+		NoDeps:              false,
+		SolverOptions:       types.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
+	}
+}
+
+type Option func(cfg *Compiler) error
+
+func (cfg *Compiler) Apply(opts ...Option) error {
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		if err := opt(cfg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WithOptions(opt *Compiler) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg = opt
+		return nil
+	}
+}
+
+func WithBackendType(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendType = r
+		return nil
+	}
+}
+
+func WithTemplateFolder(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.TemplatesFolder = r
+		return nil
+	}
+}
+
+func WithBuildValues(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BuildValuesFile = r
+		return nil
+	}
+}
+
+func WithPullRepositories(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullImageRepository = r
+		return nil
+	}
+}
+
+func WithPushRepository(r string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if len(cfg.PullImageRepository) == 0 {
+			cfg.PullImageRepository = []string{cfg.PushImageRepository}
+		}
+		cfg.PushImageRepository = r
+		return nil
+	}
+}
+
+func BackendArgs(r []string) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.BackendArgs = r
+		return nil
+	}
+}
+
+func PullFirst(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PullFirst = b
+		return nil
+	}
+}
+
+func KeepImg(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.KeepImg = b
+		return nil
+	}
+}
+
+func Rebuild(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Rebuild = b
+		return nil
+	}
+}
+
+func PushImages(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Push = b
+		return nil
+	}
+}
+
+func Wait(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Wait = b
+		return nil
+	}
+}
+
+func OnlyDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.OnlyDeps = b
+		return nil
+	}
+}
+
+func OnlyTarget(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.PackageTargetOnly = b
+		return nil
+	}
+}
+
+func NoDeps(b bool) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.NoDeps = b
+		return nil
+	}
+}
+
+func Concurrency(i int) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		if i == 0 {
+			i = runtime.NumCPU()
+		}
+		cfg.Concurrency = i
+		return nil
+	}
+}
+
+func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.CompressionType = t
+		return nil
+	}
+}
+
+func WithSolverOptions(c types.LuetSolverOptions) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.SolverOptions = c
+		return nil
+	}
+}
+
+func WithContext(c *types.Context) func(cfg *Compiler) error {
+	return func(cfg *Compiler) error {
+		cfg.Context = c
+		return nil
+	}
+}