Add helper to pre-build static images (for ARM)

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

.github/workflows/test.yml

@@ -7,6 +7,7 @@ on:
       - master
     tags:
       - '*'
+  pull_request:
 
 jobs:
   docker:
@@ -16,4 +17,4 @@ jobs:
         uses: actions/checkout@v2
       - name: Test
         run: |
-          make kind-e2e-tests
+          make kind-e2e-tests

Makefile

@@ -147,13 +147,12 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	# TODO: No need to build and then apply. `kubectl apply -k config/default` does the trick
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
-
 .PHONY: deploy-dev
 deploy-dev: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	kubectl apply -k config/dev
-	$(KUSTOMIZE) build config/dev | kubectl apply -f -
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
@@ -276,4 +275,10 @@ unit-tests: test_deps
 e2e-tests:
 	GINKGO=$(GINKGO) KUBE_VERSION=${KUBE_VERSION} $(ROOT_DIR)/script/test.sh
 
-kind-e2e-tests: ginkgo kind-setup install undeploy-dev deploy-dev e2e-tests
+kind-e2e-tests: ginkgo kind-setup install undeploy-dev deploy-dev e2e-tests
+
+
+kubesplit: manifests kustomize
+	rm -rf helm-chart
+	mkdir helm-chart
+	$(KUSTOMIZE) build config/default | kubesplit -helm helm-chart

README.md

@@ -1,9 +1,64 @@
-# osbuilder
+<h1 align="center">
+  <br>
+     <img width="184" alt="kairos-white-column 5bc2fe34" src="https://user-images.githubusercontent.com/2420543/193010398-72d4ba6e-7efe-4c2e-b7ba-d3a826a55b7d.png"><br>
+    osbuilder
+<br>
+</h1>
+
+<h3 align="center">Kubernetes Native Extension to build Linux distributions artifacts from container images</h3>
+<p align="center">
+  <a href="https://opensource.org/licenses/">
+    <img src="https://img.shields.io/badge/licence-APL2-brightgreen"
+         alt="license">
+  </a>
+  <a href="https://github.com/kairos-io/osbuilder/issues"><img src="https://img.shields.io/github/issues/kairos-io/osbuilder"></a>
+  <a href="https://kairos.io/docs/" target=_blank> <img src="https://img.shields.io/badge/Documentation-blue"
+         alt="docs"></a>
+  <img src="https://img.shields.io/badge/made%20with-Go-blue">
+  <img src="https://goreportcard.com/badge/github.com/kairos-io/osbuilder" alt="go report card" />
+</p>
+
+
+With Kairos you can build immutable, bootable Kubernetes and OS images for your edge devices as easily as writing a Dockerfile. Optional P2P mesh with distributed ledger automates node bootstrapping and coordination. Updating nodes is as easy as CI/CD: push a new image to your container registry and let secure, risk-free A/B atomic upgrades do the rest.
+
+
+<table>
+<tr>
+<th align="center">
+<img width="640" height="1px">
+<p> 
+<small>
+Documentation
+</small>
+</p>
+</th>
+<th align="center">
+<img width="640" height="1">
+<p> 
+<small>
+Contribute
+</small>
+</p>
+</th>
+</tr>
+<tr>
+<td>
+
+ 📚 [Getting started with Kairos](https://kairos.io/docs/getting-started) <br> :bulb: [Examples](https://kairos.io/docs/examples) <br> :movie_camera: [Video](https://kairos.io/docs/media/) <br> :open_hands:[Engage with the Community](https://kairos.io/community/)
+  
+</td>
+<td>
+  
+🙌[ CONTRIBUTING.md ]( https://github.com/kairos-io/kairos/blob/master/CONTRIBUTING.md ) <br> :raising_hand: [ GOVERNANCE ]( https://github.com/kairos-io/kairos/blob/master/GOVERNANCE.md ) <br>:construction_worker:[Code of conduct](https://github.com/kairos-io/kairos/blob/master/CODE_OF_CONDUCT.md) 
+  
+</td>
+</tr>
+</table>
 
 | :exclamation: | This is experimental! |
 |-|:-|
 
-This is the Kairos osbuilder Kubernetes Native Extension. 
+## Installation
 
 To install, use helm:
 
@@ -28,3 +83,5 @@ TEST SUITE: None
 # Installs osbuilder
 $ helm install kairos-osbuilder kairos/osbuilder
 ```
+
+Documentation available at: https://kairos.io/docs/advanced/build/

api/v1alpha1/osartifact_types.go

@@ -30,7 +30,18 @@ type OSArtifactSpec struct {
 
 	// Foo is an example field of OSArtifact. Edit osartifact_types.go to remove/update
 	ImageName string `json:"imageName,omitempty"`
-	ISO       bool   `json:"iso,omitempty"`
+	// This needs to be revisited
+	ISO bool `json:"iso,omitempty"`
+
+	//Disk-only stuff
+	DiskSize   string `json:"diskSize,omitempty"`
+	CloudImage bool   `json:"cloudImage,omitempty"`
+	AzureImage bool   `json:"azureImage,omitempty"`
+	GCEImage   bool   `json:"gceImage,omitempty"`
+
+	Netboot    bool   `json:"netboot,omitempty"`
+	NetbootURL string `json:"netbootURL,omitempty"`
+
 	// TODO: treat cloudconfig as a secret, and take a secretRef where to store it (optionally)
 	CloudConfig string `json:"cloudConfig,omitempty"`
 	GRUBConfig  string `json:"grubConfig,omitempty"`
@@ -38,7 +49,8 @@ type OSArtifactSpec struct {
 	Bundles     []string `json:"bundles,omitempty"`
 	PullOptions Pull     `json:"pull,omitempty"`
 	OSRelease   string   `json:"osRelease,omitempty"`
-	PushOptions Push     `json:"push,omitempty"`
+	// TODO: Currently not used. Reserved to be used when we have a way to push to registries.
+	PushOptions Push `json:"push,omitempty"`
 }
 
 type Push struct {

config/crd/bases/build.kairos.io_osartifacts.yaml

@@ -35,6 +35,8 @@ spec:
           spec:
             description: OSArtifactSpec defines the desired state of OSArtifact
             properties:
+              azureImage:
+                type: boolean
               bundles:
                 items:
                   type: string
@@ -43,6 +45,13 @@ spec:
                 description: 'TODO: treat cloudconfig as a secret, and take a secretRef
                   where to store it (optionally)'
                 type: string
+              cloudImage:
+                type: boolean
+              diskSize:
+                description: Disk-only stuff
+                type: string
+              gceImage:
+                type: boolean
               grubConfig:
                 type: string
               imageName:
@@ -50,7 +59,12 @@ spec:
                   to remove/update
                 type: string
               iso:
+                description: This needs to be revisited
                 type: boolean
+              netboot:
+                type: boolean
+              netbootURL:
+                type: string
               osRelease:
                 type: string
               pull:

config/default/kustomization.yaml

@@ -16,6 +16,7 @@ bases:
 - ../crd
 - ../rbac
 - ../manager
+- ../nginx
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- ../webhook
@@ -72,3 +73,16 @@ vars:
 #    kind: Service
 #    version: v1
 #    name: webhook-service
+
+vars:
+- name: NGINX_NAMESPACE
+  objref:
+    kind: Namespace
+    name: system
+    apiVersion: v1 
+
+- name: ARTIFACT_COPIER_ROLE
+  objref:
+    kind: Role
+    name: artifactCopier
+    apiVersion: rbac.authorization.k8s.io/v1

config/default/manager_auth_proxy_patch.yaml

@@ -38,3 +38,7 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        - "--copy-to-namespace=$(NGINX_NAMESPACE)"
+        - "--copy-role=$(ARTIFACT_COPIER_ROLE)"
+        - --copy-to-pod-label=app.kubernetes.io/name=osbuilder-nginx
+        - --copy-to-path="/usr/share/nginx/html"

config/dev/kustomization.yaml

@@ -1,74 +1,7 @@
-# Adds namespace to all resources.
-namespace: osartifactbuilder-operator-system
-
-# Value of this field is prepended to the
-# names of all resources, e.g. a deployment named
-# "wordpress" becomes "alices-wordpress".
-# Note that it should also match with the prefix (text before '-') of the namespace
-# field above.
-namePrefix: osartifactbuilder-operator-
-
-# Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
-
 bases:
-- ../crd
+- ../default
-- ../rbac
-- ../manager
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
-#- ../webhook
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-#- ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
 
-patchesStrategicMerge:
+images:
-# Protect the /metrics endpoint by putting it behind auth.
+- name: quay.io/kairos/osbuilder
-# If you want your controller-manager to expose the /metrics
+  newName: quay.io/kairos/osbuilder
-# endpoint w/o any authn/z, please comment the following line.
+  newTag: test
-- manager_auth_proxy_patch.yaml
-
-# Mount the controller config file for loading manager configurations
-# through a ComponentConfig type
-#- manager_config_patch.yaml
-
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
-# crd/kustomization.yaml
-#- manager_webhook_patch.yaml
-
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
-# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
-# 'CERTMANAGER' needs to be enabled to use ca injection
-#- webhookcainjection_patch.yaml
-
-# the following config is for teaching kustomize how to do var substitution
-vars:
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: CERTIFICATE_NAME
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#- name: SERVICE_NAMESPACE # namespace of the service
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: SERVICE_NAME
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service

config/dev/manager_auth_proxy_patch.yaml

@@ -1,40 +0,0 @@
-# This patch inject a sidecar container which is a HTTP proxy for the
-# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: kube-rbac-proxy
-        securityContext:
-          allowPrivilegeEscalation: false
-        # TODO(user): uncomment for common cases that do not require escalating privileges
-        # capabilities:
-        #   drop:
-        #     - "ALL"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.11.0
-        args:
-        - "--secure-listen-address=0.0.0.0:8443"
-        - "--upstream=http://127.0.0.1:8080/"
-        - "--logtostderr=true"
-        - "--v=0"
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-        resources:
-          limits:
-            cpu: 500m
-            memory: 128Mi
-          requests:
-            cpu: 5m
-            memory: 64Mi
-      - name: manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"

config/dev/manager_config_patch.yaml

@@ -1,21 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: controller-manager
-  namespace: system
-spec:
-  template:
-    spec:
-      containers:
-      - name: manager
-        imagePullPolicy: Never
-        args:
-        - "--config=controller_manager_config.yaml"
-        volumeMounts:
-        - name: manager-config
-          mountPath: /controller_manager_config.yaml
-          subPath: controller_manager_config.yaml
-      volumes:
-      - name: manager-config
-        configMap:
-          name: manager-config

config/manager/kustomization.yaml

@@ -1,6 +1,8 @@
 resources:
 - manager.yaml
 
+namespace: system
+
 generatorOptions:
   disableNameSuffixHash: true
 

config/manager/manager.yaml

@@ -9,7 +9,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: controller-manager
-  namespace: system
   labels:
     control-plane: controller-manager
 spec:

config/nginx/deployment.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nginx-public
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 3Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app.kubernetes.io/name: osbuilder-nginx
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: osbuilder-nginx
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: osbuilder-nginx
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+        volumeMounts:
+          - mountPath: "/usr/share/nginx/html"
+            name: nginx-public
+        ports:
+          - containerPort: 80
+      serviceAccountName: controller-manager
+      terminationGracePeriodSeconds: 10
+      volumes:
+        - name: nginx-public
+          persistentVolumeClaim:
+            claimName: nginx-public

config/nginx/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- deployment.yaml
+- service.yaml
+- role.yaml

config/nginx/role.yaml

@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: artifactCopier
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create

config/nginx/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: osbuilder-nginx
+spec:
+  type: NodePort
+  selector:
+    app.kubernetes.io/name: osbuilder-nginx
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80

config/rbac/role_custom.yaml

@@ -5,6 +5,31 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - roles
+  - rolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - build.kairos.io
   resources:
@@ -40,17 +65,30 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - services
   - configmaps
   verbs:
   - get
   - create
   - update
 - apiGroups:
-  - "apps"
+  - "batch"
   resources:
-  - deployments
+  - jobs
   verbs:
   - get
   - create
   - update
+# Temporary so that it can grant these permissions to the created role
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create

controllers/deployment.go

@@ -1,224 +0,0 @@
-/*
-Copyright 2022.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controllers
-
-import (
-	"fmt"
-
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	appsv1 "k8s.io/api/apps/v1"
-	v1 "k8s.io/api/core/v1"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func genDeploymentLabel(s string) map[string]string {
-	return map[string]string{
-		"osbuild": "workload" + s,
-	}
-}
-
-// TODO: Handle registry auth
-// TODO: This shells out, but needs ENV_VAR with key refs mapping
-func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1alpha1.Pull) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		Name:            fmt.Sprintf("pull-image-%s", id),
-		Image:           containerImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			fmt.Sprintf(
-				"luet util unpack %s %s",
-				pullImage,
-				"/rootfs",
-			),
-		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-		},
-	}
-}
-
-func createImageContainer(containerImage string, pushOptions buildv1alpha1.Push) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		Name:            "create-image",
-		Image:           containerImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			fmt.Sprintf(
-				"tar -czvpf test.tar -C /rootfs . && luet util pack %s test.tar image.tar && mv image.tar /public",
-				pushOptions.ImageName,
-			),
-		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-			{
-				Name:      "public",
-				MountPath: "/public",
-			},
-		},
-	}
-}
-
-func osReleaseContainer(containerImage string) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		Name:            "os-release",
-		Image:           containerImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			"cp -rfv /etc/os-release /rootfs/etc/os-release",
-		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "config",
-				MountPath: "/etc/os-release",
-				SubPath:   "os-release",
-			},
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-		},
-	}
-}
-
-func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact) *appsv1.Deployment {
-	objMeta := metav1.ObjectMeta{
-		Name:            artifact.Name,
-		Namespace:       artifact.Namespace,
-		OwnerReferences: genOwner(artifact),
-	}
-
-	pushImage := artifact.Spec.PushOptions.Push
-
-	privileged := false
-	serviceAccount := false
-
-	buildIsoContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "build-iso",
-		Image:           r.ToolImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			fmt.Sprintf(
-				"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /public dir:/rootfs",
-				artifact.Name,
-			),
-		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "public",
-				MountPath: "/public",
-			},
-			{
-				Name:      "config",
-				MountPath: "/iso/iso-overlay/cloud_config.yaml",
-				SubPath:   "config",
-			},
-			{
-				Name:      "config",
-				MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
-				SubPath:   "grub.cfg",
-			},
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-		},
-	}
-
-	servingContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "serve",
-		Ports:           []v1.ContainerPort{v1.ContainerPort{Name: "http", ContainerPort: 80}},
-		Image:           r.ServingImage,
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "public",
-				MountPath: "/usr/share/nginx/html",
-			},
-		},
-	}
-
-	pod := v1.PodSpec{
-		AutomountServiceAccountToken: &serviceAccount,
-		Volumes: []v1.Volume{
-			{
-				Name:         "public",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
-			},
-			{
-				Name:         "rootfs",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
-			},
-			{
-				Name: "config",
-				VolumeSource: v1.VolumeSource{
-					ConfigMap: &v1.ConfigMapVolumeSource{
-						LocalObjectReference: v1.LocalObjectReference{Name: artifact.Name}}},
-			},
-		},
-	}
-
-	pod.InitContainers = []v1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName, artifact.Spec.PullOptions)}
-
-	for i, bundle := range artifact.Spec.Bundles {
-		pod.InitContainers = append(pod.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle, artifact.Spec.PullOptions))
-	}
-
-	if artifact.Spec.OSRelease != "" {
-		pod.InitContainers = append(pod.InitContainers, osReleaseContainer(r.ToolImage))
-
-	}
-
-	pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
-
-	if pushImage {
-		pod.InitContainers = append(pod.InitContainers, createImageContainer(r.ToolImage, artifact.Spec.PushOptions))
-
-	}
-
-	pod.Containers = []v1.Container{servingContainer}
-
-	deploymentLabels := genDeploymentLabel(artifact.Name)
-	replicas := int32(1)
-
-	return &appsv1.Deployment{
-		ObjectMeta: objMeta,
-
-		Spec: appsv1.DeploymentSpec{
-			Selector: &metav1.LabelSelector{MatchLabels: deploymentLabels},
-			Replicas: &replicas,
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: deploymentLabels,
-				},
-				Spec: pod,
-			},
-		},
-	}
-}

controllers/job.go

@@ -0,0 +1,533 @@
+/*
+Copyright 2022.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
+	"github.com/pkg/errors"
+	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+func genJobLabel(s string) map[string]string {
+	return map[string]string{
+		"osbuild": "workload" + s,
+	}
+}
+
+// TODO: Handle registry auth
+// TODO: This shells out, but needs ENV_VAR with key refs mapping
+// TODO: Cache downloaded images?
+func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1alpha1.Pull) v1.Container {
+	return v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		Name:            fmt.Sprintf("pull-image-%s", id),
+		Image:           containerImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			fmt.Sprintf(
+				"luet util unpack %s %s",
+				pullImage,
+				"/rootfs",
+			),
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "rootfs",
+				MountPath: "/rootfs",
+			},
+		},
+	}
+}
+
+func pushImageName(artifact buildv1alpha1.OSArtifact) string {
+	pushName := artifact.Spec.PushOptions.ImageName
+	if pushName != "" {
+		return pushName
+	}
+	return artifact.Name
+}
+
+func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifact) v1.Container {
+	imageName := pushImageName(artifact)
+
+	return v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		Name:            "create-image",
+		Image:           containerImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			fmt.Sprintf(
+				"tar -czvpf test.tar -C /rootfs . && luet util pack %[1]s test.tar %[2]s.tar && chmod +r %[2]s.tar && mv %[2]s.tar /artifacts",
+				imageName,
+				artifact.Name,
+			),
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "rootfs",
+				MountPath: "/rootfs",
+			},
+			{
+				Name:      "artifacts",
+				MountPath: "/artifacts",
+			},
+		},
+	}
+}
+
+func createPushToServerImageContainer(containerImage string, artifactPodInfo ArtifactPodInfo) v1.Container {
+	command := fmt.Sprintf("tar cf - -C artifacts/ . | kubectl exec -i -n %s $(kubectl  get pods -l %s -n %s --no-headers -o custom-columns=\":metadata.name\" | head -n1) -- tar xf - -C %s", artifactPodInfo.Namespace, artifactPodInfo.Label, artifactPodInfo.Namespace, artifactPodInfo.Path)
+	fmt.Printf("command = %+v\n", command)
+
+	return v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		Name:            "push-to-server",
+		Image:           containerImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args:            []string{command},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "rootfs",
+				MountPath: "/rootfs",
+			},
+			{
+				Name:      "artifacts",
+				MountPath: "/artifacts",
+			},
+		},
+	}
+}
+
+func osReleaseContainer(containerImage string) v1.Container {
+	return v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		Name:            "os-release",
+		Image:           containerImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			"cp -rfv /etc/os-release /rootfs/etc/os-release",
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "config",
+				MountPath: "/etc/os-release",
+				SubPath:   "os-release",
+			},
+			{
+				Name:      "rootfs",
+				MountPath: "/rootfs",
+			},
+		},
+	}
+}
+
+func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv1.Job {
+	objMeta := genObjectMeta(artifact)
+
+	privileged := false
+	serviceAccount := true
+
+	cmd := fmt.Sprintf(
+		"/entrypoint.sh --debug --name %s build-iso --date=false --output /artifacts dir:/rootfs",
+		artifact.Name,
+	)
+
+	volumeMounts := []v1.VolumeMount{
+		{
+			Name:      "artifacts",
+			MountPath: "/artifacts",
+		},
+		{
+			Name:      "rootfs",
+			MountPath: "/rootfs",
+		},
+	}
+
+	if artifact.Spec.GRUBConfig != "" {
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      "config",
+			MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
+			SubPath:   "grub.cfg",
+		})
+	}
+
+	cloudImgCmd := fmt.Sprintf(
+		"/raw-images.sh /rootfs /artifacts/%s.raw",
+		artifact.Name,
+	)
+
+	if artifact.Spec.CloudConfig != "" {
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      "config",
+			MountPath: "/iso/iso-overlay/cloud_config.yaml",
+			SubPath:   "config",
+		})
+
+		cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
+	}
+
+	if artifact.Spec.CloudConfig != "" || artifact.Spec.GRUBConfig != "" {
+		cmd = fmt.Sprintf(
+			"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /artifacts dir:/rootfs",
+			artifact.Name,
+		)
+	}
+
+	buildIsoContainer := v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+		Name:            "build-iso",
+		Image:           r.ToolImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			cmd,
+		},
+		VolumeMounts: volumeMounts,
+	}
+
+	buildCloudImageContainer := v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+		Name:            "build-cloud-image",
+		Image:           r.ToolImage,
+
+		Command: []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			cloudImgCmd,
+		},
+		VolumeMounts: volumeMounts,
+	}
+
+	if artifact.Spec.DiskSize != "" {
+		buildCloudImageContainer.Env = []v1.EnvVar{{
+			Name:  "EXTEND",
+			Value: artifact.Spec.DiskSize,
+		}}
+	}
+
+	extractNetboot := v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+		Name:            "build-netboot",
+		Image:           r.ToolImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Env: []v1.EnvVar{{
+			Name:  "URL",
+			Value: artifact.Spec.NetbootURL,
+		}},
+		Args: []string{
+			fmt.Sprintf(
+				"/netboot.sh /artifacts/%s.iso /artifacts/%s",
+				artifact.Name,
+				artifact.Name,
+			),
+		},
+		VolumeMounts: volumeMounts,
+	}
+
+	buildAzureCloudImageContainer := v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+		Name:            "build-azure-cloud-image",
+		Image:           r.ToolImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			fmt.Sprintf(
+				"/azure.sh /artifacts/%s.raw /artifacts/%s.vhd",
+				artifact.Name,
+				artifact.Name,
+			),
+		},
+		VolumeMounts: volumeMounts,
+	}
+
+	buildGCECloudImageContainer := v1.Container{
+		ImagePullPolicy: v1.PullAlways,
+		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+		Name:            "build-gce-cloud-image",
+		Image:           r.ToolImage,
+		Command:         []string{"/bin/bash", "-cxe"},
+		Args: []string{
+			fmt.Sprintf(
+				"/gce.sh /artifacts/%s.raw /artifacts/%s.gce.raw",
+				artifact.Name,
+				artifact.Name,
+			),
+		},
+		VolumeMounts: volumeMounts,
+	}
+
+	pod := v1.PodSpec{
+		AutomountServiceAccountToken: &serviceAccount,
+		ServiceAccountName:           objMeta.Name,
+		RestartPolicy:                v1.RestartPolicyNever,
+		Volumes: []v1.Volume{
+			{
+				Name:         "artifacts",
+				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+			},
+			{
+				Name:         "rootfs",
+				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+			},
+			{
+				Name: "config",
+				VolumeSource: v1.VolumeSource{
+					ConfigMap: &v1.ConfigMapVolumeSource{
+						LocalObjectReference: v1.LocalObjectReference{Name: artifact.Name}}},
+			},
+		},
+	}
+
+	pod.InitContainers = []v1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName, artifact.Spec.PullOptions)}
+
+	for i, bundle := range artifact.Spec.Bundles {
+		pod.InitContainers = append(pod.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle, artifact.Spec.PullOptions))
+	}
+
+	if artifact.Spec.OSRelease != "" {
+		pod.InitContainers = append(pod.InitContainers, osReleaseContainer(r.ToolImage))
+	}
+
+	if artifact.Spec.ISO || artifact.Spec.Netboot {
+		pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
+	}
+
+	if artifact.Spec.Netboot {
+		pod.InitContainers = append(pod.InitContainers, extractNetboot)
+	}
+
+	if artifact.Spec.CloudImage || artifact.Spec.AzureImage || artifact.Spec.GCEImage {
+		pod.InitContainers = append(pod.InitContainers, buildCloudImageContainer)
+	}
+
+	if artifact.Spec.AzureImage {
+		pod.InitContainers = append(pod.InitContainers, buildAzureCloudImageContainer)
+	}
+
+	if artifact.Spec.GCEImage {
+		pod.InitContainers = append(pod.InitContainers, buildGCECloudImageContainer)
+	}
+
+	pod.InitContainers = append(pod.InitContainers, createImageContainer(r.ToolImage, artifact))
+
+	pod.Containers = []v1.Container{
+		createPushToServerImageContainer(r.CopierImage, r.ArtifactPodInfo),
+	}
+
+	jobLabels := genJobLabel(artifact.Name)
+
+	job := batchv1.Job{
+		ObjectMeta: objMeta,
+		Spec: batchv1.JobSpec{
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: jobLabels,
+				},
+				Spec: pod,
+			},
+		},
+	}
+
+	return &job
+}
+
+// createServiceAccount creates a service account that has the permissions to
+// copy the artifacts to the http server Pod. This service account is used for
+// the "push to server" container.
+func (r *OSArtifactReconciler) createCopierServiceAccount(ctx context.Context, objMeta metav1.ObjectMeta) error {
+	sa, err := r.clientSet.CoreV1().
+		ServiceAccounts(objMeta.Namespace).Get(ctx, objMeta.Name, metav1.GetOptions{})
+	if sa == nil || apierrors.IsNotFound(err) {
+		t := true
+		_, err := r.clientSet.CoreV1().ServiceAccounts(objMeta.Namespace).Create(ctx,
+			&v1.ServiceAccount{
+				ObjectMeta:                   objMeta,
+				AutomountServiceAccountToken: &t,
+			}, metav1.CreateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+// func (r *OSArtifactReconciler) createCopierRole(ctx context.Context, objMeta metav1.ObjectMeta) error {
+// 	role, err := r.clientSet.RbacV1().
+// 		Roles(objMeta.Namespace).
+// 		Get(ctx, objMeta.Name, metav1.GetOptions{})
+// 	if role == nil || apierrors.IsNotFound(err) {
+// 		_, err := r.clientSet.RbacV1().Roles(objMeta.Namespace).Create(ctx,
+// 			&rbacv1.Role{
+// 				ObjectMeta: objMeta,
+// 				Rules: []rbacv1.PolicyRule{
+// 					// TODO: The actual permissions we need is that to copy to a Pod.
+// 					// The Pod is on another namespace, so we need a cluster wide permission.
+// 					// This can get viral because the controller needs to have the permissions
+// 					// if it is to grant them to the Job.
+// 					{
+// 						Verbs:     []string{"list"},
+// 						APIGroups: []string{""},
+// 						Resources: []string{"pods"},
+// 					},
+// 				},
+// 			},
+// 			metav1.CreateOptions{},
+// 		)
+// 		if err != nil {
+// 			return err
+// 		}
+// 	}
+
+// 	return err
+// }
+
+func (r *OSArtifactReconciler) createCopierRoleBinding(ctx context.Context, objMeta metav1.ObjectMeta) error {
+	newrb := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      objMeta.Name,
+			Namespace: r.ArtifactPodInfo.Namespace,
+			// TODO: We can't have cross-namespace owners. The role binding will have to deleted explicitly by the reconciler (finalizer?)
+			// OwnerReferences: objMeta.OwnerReferences,
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "Role",
+			Name:     r.ArtifactPodInfo.Role,
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      objMeta.Name,
+				Namespace: objMeta.Namespace,
+			},
+		},
+	}
+
+	rb, err := r.clientSet.RbacV1().
+		RoleBindings(r.ArtifactPodInfo.Namespace).
+		Get(ctx, objMeta.Name, metav1.GetOptions{})
+	if rb == nil || apierrors.IsNotFound(err) {
+		_, err := r.clientSet.RbacV1().
+			RoleBindings(r.ArtifactPodInfo.Namespace).
+			Create(ctx, newrb, metav1.CreateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+// createRBAC creates a ServiceAccount, and a binding to the CopierRole so that
+// the container that copies the artifacts to the http server Pod has the
+// permissions to do so.
+func (r *OSArtifactReconciler) createRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
+	objMeta := genObjectMeta(artifact)
+
+	err := r.createCopierServiceAccount(ctx, objMeta)
+	if err != nil {
+		return errors.Wrap(err, "creating a service account")
+	}
+
+	err = r.createCopierRoleBinding(ctx, objMeta)
+	if err != nil {
+		return errors.Wrap(err, "creating a role binding for the copy-role")
+	}
+
+	return err
+}
+
+// removeRBAC deletes the role binding between the service account of this artifact
+// and the CopierRole. The ServiceAccount is removed automatically through the Owner
+// relationship with the OSArtifact. The RoleBinding can't have it as an owner
+// because it is in a different Namespace.
+func (r *OSArtifactReconciler) removeRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
+	err := r.clientSet.RbacV1().RoleBindings(r.ArtifactPodInfo.Namespace).
+		Delete(ctx, artifact.Name, metav1.DeleteOptions{})
+	// Ignore not found. No need to do anything.
+	if err != nil && apierrors.IsNotFound(err) {
+		return nil
+	}
+
+	return err
+}
+
+func (r *OSArtifactReconciler) removeArtifacts(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
+	//Finding Pods using labels
+	fmt.Printf("r.ArtifactPodInfo = %+v\n", r.ArtifactPodInfo.Label)
+	pods, err := r.clientSet.CoreV1().Pods(r.ArtifactPodInfo.Namespace).
+		List(ctx, metav1.ListOptions{LabelSelector: r.ArtifactPodInfo.Label})
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("listing pods with label %s in namespace %s", r.ArtifactPodInfo.Label, r.ArtifactPodInfo.Namespace))
+	}
+	if len(pods.Items) < 1 {
+		return errors.New("No artifact pod found")
+	}
+	pod := pods.Items[0]
+
+	stdout, stderr, err := r.executeRemoteCommand(r.ArtifactPodInfo.Namespace, pod.Name, fmt.Sprintf("rm -rf %s/%s.*", r.ArtifactPodInfo.Path, artifact.Name))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("%s\n%s", stdout, stderr))
+	}
+	return nil
+}
+
+func (r *OSArtifactReconciler) executeRemoteCommand(namespace, podName, command string) (string, string, error) {
+	buf := &bytes.Buffer{}
+	errBuf := &bytes.Buffer{}
+	request := r.clientSet.CoreV1().RESTClient().
+		Post().
+		Namespace(namespace).
+		Resource("pods").
+		Name(podName).
+		SubResource("exec").
+		VersionedParams(&v1.PodExecOptions{
+			Command: []string{"/bin/sh", "-c", command},
+			Stdin:   false,
+			Stdout:  true,
+			Stderr:  true,
+			TTY:     true,
+		}, scheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(r.restConfig, "POST", request.URL())
+	if err != nil {
+		return "", "", err
+	}
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdout: buf,
+		Stderr: errBuf,
+	})
+	if err != nil {
+		return "", "", fmt.Errorf("%w Failed executing command %s on %v/%v", err, command, namespace, podName)
+	}
+
+	return buf.String(), errBuf.String(), nil
+}

controllers/osartifact_controller.go

@@ -25,22 +25,42 @@ import (
 	"github.com/pkg/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+const FinalizerName = "build.kairos.io/osbuilder-finalizer"
+
+type ArtifactPodInfo struct {
+	Label     string
+	Namespace string
+	Path      string
+	Role      string
+}
+
 // OSArtifactReconciler reconciles a OSArtifact object
 type OSArtifactReconciler struct {
 	client.Client
-	Scheme                  *runtime.Scheme
+	Scheme                               *runtime.Scheme
-	clientSet               *kubernetes.Clientset
+	restConfig                           *rest.Config
-	ServingImage, ToolImage string
+	clientSet                            *kubernetes.Clientset
+	ServingImage, ToolImage, CopierImage string
+	ArtifactPodInfo                      ArtifactPodInfo
+}
+
+func genObjectMeta(artifact buildv1alpha1.OSArtifact) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Name:            artifact.Name,
+		Namespace:       artifact.Namespace,
+		OwnerReferences: genOwner(artifact),
+	}
 }
 
 func genOwner(artifact buildv1alpha1.OSArtifact) []metav1.OwnerReference {
@@ -57,6 +77,12 @@ func genOwner(artifact buildv1alpha1.OSArtifact) []metav1.OwnerReference {
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/finalizers,verbs=update
 
+// TODO: Is this ^ how I should have created rbac permissions for the controller?
+//       - git commit all changes
+//       - generate code with kubebuilder
+//       - check if my permissions were removed
+//       - do it properly
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 // TODO(user): Modify the Reconcile function to compare the state specified by
@@ -79,17 +105,22 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	logger.Info(fmt.Sprintf("Reconciling %v", osbuild))
 
+	stop, err := r.handleFinalizer(ctx, &osbuild)
+	if err != nil || stop {
+		return ctrl.Result{}, err
+	}
+
 	// generate configmap required for building a custom image
 	desiredConfigMap := r.genConfigMap(osbuild)
 	logger.Info(fmt.Sprintf("Checking configmap %v", osbuild))
 
-	cfgMap, err := r.clientSet.CoreV1().ConfigMaps(req.Namespace).Get(ctx, desiredConfigMap.Name, v1.GetOptions{})
+	cfgMap, err := r.clientSet.CoreV1().ConfigMaps(req.Namespace).Get(ctx, desiredConfigMap.Name, metav1.GetOptions{})
 	if cfgMap == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating service %v", desiredConfigMap))
+		logger.Info(fmt.Sprintf("Creating config map %v", desiredConfigMap))
 
-		cfgMap, err = r.clientSet.CoreV1().ConfigMaps(req.Namespace).Create(ctx, desiredConfigMap, v1.CreateOptions{})
+		_, err = r.clientSet.CoreV1().ConfigMaps(req.Namespace).Create(ctx, desiredConfigMap, metav1.CreateOptions{})
 		if err != nil {
-			logger.Error(err, "Failed while creating svc")
+			logger.Error(err, "Failed while creating config map")
 			return ctrl.Result{}, err
 		}
 		return ctrl.Result{Requeue: true}, err
@@ -98,34 +129,21 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	desiredService := genService(osbuild)
-	logger.Info(fmt.Sprintf("Checking service %v", osbuild))
-
-	svc, err := r.clientSet.CoreV1().Services(req.Namespace).Get(ctx, desiredService.Name, v1.GetOptions{})
-	if svc == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating service %v", desiredService))
-
-		svc, err = r.clientSet.CoreV1().Services(req.Namespace).Create(ctx, desiredService, v1.CreateOptions{})
-		if err != nil {
-			logger.Error(err, "Failed while creating svc")
-			return ctrl.Result{}, err
-		}
-
-		return ctrl.Result{Requeue: true}, err
-	}
-	if err != nil {
-		return ctrl.Result{Requeue: true}, err
-	}
 	logger.Info(fmt.Sprintf("Checking deployment %v", osbuild))
 
-	desiredDeployment := r.genDeployment(osbuild)
+	err = r.createRBAC(ctx, osbuild)
-	deployment, err := r.clientSet.AppsV1().Deployments(req.Namespace).Get(ctx, desiredDeployment.Name, v1.GetOptions{})
+	if err != nil {
-	if deployment == nil || apierrors.IsNotFound(err) {
+		return ctrl.Result{Requeue: true}, err
-		logger.Info(fmt.Sprintf("Creating Deployment %v", deployment))
+	}
 
-		deployment, err = r.clientSet.AppsV1().Deployments(req.Namespace).Create(ctx, desiredDeployment, v1.CreateOptions{})
+	desiredJob := r.genJob(osbuild)
+	job, err := r.clientSet.BatchV1().Jobs(req.Namespace).Get(ctx, desiredJob.Name, metav1.GetOptions{})
+	if job == nil || apierrors.IsNotFound(err) {
+		logger.Info(fmt.Sprintf("Creating Job %v", job))
+
+		_, err = r.clientSet.BatchV1().Jobs(req.Namespace).Create(ctx, desiredJob, metav1.CreateOptions{})
 		if err != nil {
-			logger.Error(err, "Failed while creating deployment")
+			logger.Error(err, "Failed while creating job")
 			return ctrl.Result{}, nil
 		}
 
@@ -143,7 +161,7 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if deployment.Status.ReadyReplicas == deployment.Status.Replicas {
+	if job.Status.Succeeded > 0 {
 		copy.Status.Phase = "Ready"
 	} else if copy.Status.Phase != "Building" {
 		copy.Status.Phase = "Building"
@@ -166,13 +184,66 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 // SetupWithManager sets up the controller with the Manager.
 func (r *OSArtifactReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
-	clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
+	cfg := mgr.GetConfig()
+	clientset, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		return err
 	}
+	r.restConfig = cfg
 	r.clientSet = clientset
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&buildv1alpha1.OSArtifact{}).
 		Complete(r)
 }
+
+// Returns true if reconciliation should stop or false otherwise
+func (r *OSArtifactReconciler) handleFinalizer(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) (bool, error) {
+	// examine DeletionTimestamp to determine if object is under deletion
+	if osbuild.DeletionTimestamp.IsZero() {
+		// The object is not being deleted, so if it does not have our finalizer,
+		// then lets add the finalizer and update the object. This is equivalent
+		// registering our finalizer.
+		if !controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
+			controllerutil.AddFinalizer(osbuild, FinalizerName)
+			if err := r.Update(ctx, osbuild); err != nil {
+				return true, err
+			}
+		}
+	} else {
+		// The object is being deleted
+		if controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
+			// our finalizer is present, so lets handle any external dependency
+			if err := r.finalize(ctx, osbuild); err != nil {
+				// if fail to delete the external dependency here, return with error
+				// so that it can be retried
+				return true, err
+			}
+
+			// remove our finalizer from the list and update it.
+			controllerutil.RemoveFinalizer(osbuild, FinalizerName)
+			if err := r.Update(ctx, osbuild); err != nil {
+				return true, err
+			}
+		}
+
+		// Stop reconciliation as the item is being deleted
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// - Remove artifacts from the server Pod
+// - Delete role-binding (because it doesn't have the OSArtifact as an owner and won't be deleted automatically)
+func (r *OSArtifactReconciler) finalize(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) error {
+	if err := r.removeRBAC(ctx, *osbuild); err != nil {
+		return err
+	}
+
+	if err := r.removeArtifacts(ctx, *osbuild); err != nil {
+		return err
+	}
+
+	return nil
+}

controllers/service.go

@@ -1,39 +0,0 @@
-/*
-Copyright 2022.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controllers
-
-import (
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func genService(artifact buildv1alpha1.OSArtifact) *v1.Service {
-	objMeta := metav1.ObjectMeta{
-		Name:            artifact.Name,
-		Namespace:       artifact.Namespace,
-		OwnerReferences: genOwner(artifact),
-	}
-	return &v1.Service{
-		ObjectMeta: objMeta,
-		Spec: v1.ServiceSpec{
-			Type:     v1.ServiceTypeNodePort,
-			Ports:    []v1.ServicePort{{Name: "http", Port: int32(80)}},
-			Selector: genDeploymentLabel(artifact.Name),
-		},
-	}
-}

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

go.sum

@@ -82,6 +82,7 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.m
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -136,6 +137,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
@@ -189,6 +191,7 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I=
 github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8=
@@ -277,6 +280,7 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -373,6 +377,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -826,6 +831,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y=
 golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -52,12 +52,22 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
-	var serveImage, toolImage string
+	var serveImage, toolImage, copierImage string
+	var copyToPodLabel, copyToNamespace, copyToPath, copierRole string
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+
+	flag.StringVar(&copierImage, "copier-image", "quay.io/kairos/kubectl", "The image that is used to copy artifacts to the server pod.")
 	flag.StringVar(&serveImage, "serve-image", "nginx", "Serve image.")
 	// It needs luet inside
 	flag.StringVar(&toolImage, "tool-image", "quay.io/kairos/osbuilder-tools:latest", "Tool image.")
 
+	// Information on where to copy the artifacts
+	flag.StringVar(&copyToPodLabel, "copy-to-pod-label", "", "The label of the Pod to which artifacts should be copied.")
+	flag.StringVar(&copyToNamespace, "copy-to-namespace", "", "The namespace of the copy-to-pod-label Pod.")
+	flag.StringVar(&copyToPath, "copy-to-path", "", "The path under which to copy artifacts in the copy-to-pod-label Pod.")
+	flag.StringVar(&copierRole, "copy-role", "", "The name or the Kubernetes Role that has the permissions to copy artifacts to the copy-to-pod-label Pod")
+
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -98,7 +108,14 @@ func main() {
 		Client:       mgr.GetClient(),
 		ServingImage: serveImage,
 		ToolImage:    toolImage,
-		Scheme:       mgr.GetScheme(),
+		CopierImage:  copierImage,
+		ArtifactPodInfo: controllers.ArtifactPodInfo{
+			Label:     copyToPodLabel,
+			Namespace: copyToNamespace,
+			Path:      copyToPath,
+			Role:      copierRole,
+		},
+		Scheme: mgr.GetScheme(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "OSArtifact")
 		os.Exit(1)

tests/e2e/e2e_simple_test.go

@@ -1,6 +1,10 @@
 package e2e_test
 
 import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -20,20 +24,91 @@ var _ = Describe("ISO build test", func() {
 			err := kubectl.Apply("", "../../tests/fixtures/simple.yaml")
 			Expect(err).ToNot(HaveOccurred())
 
-			Eventually(func() string {
+			itHasTheCorrectImage()
-				b, _ := kubectl.GetData("default", "osartifacts", "hello-kairos", "jsonpath={.spec.imageName}")
+			itHasTheCorrectLabels()
-				return string(b)
+			itCopiesTheArtifacts()
-			}, 2*time.Minute, 2*time.Second).Should(Equal("quay.io/kairos/core-opensuse:latest"))
 
-			Eventually(func() string {
+			By("deleting the custom resource", func() {
-				b, _ := kubectl.GetData("default", "deployments", "hello-kairos", "jsonpath={.spec.template.metadata.labels.osbuild}")
+				err = kubectl.New().Delete("osartifacts", "-n", "default", "hello-kairos")
-				return string(b)
+				Expect(err).ToNot(HaveOccurred())
-			}, 2*time.Minute, 2*time.Second).Should(Equal("workloadhello-kairos"))
+			})
-			Eventually(func() string {
+
-				b, _ := kubectl.GetData("default", "deployments", "hello-kairos", "jsonpath={.spec.status.unavailableReplicas}")
+			itCleansUpRoleBindings()
-				return string(b)
+			itDeletesTheArtifacts()
-			}, 15*time.Minute, 2*time.Second).ShouldNot(Equal("1"))
 		})
 	})
-
 })
+
+func itHasTheCorrectImage() {
+	Eventually(func() string {
+		b, _ := kubectl.GetData("default", "osartifacts", "hello-kairos", "jsonpath={.spec.imageName}")
+		fmt.Printf("looking for image core-opensuse:latest = %+v\n", string(b))
+		return string(b)
+	}, 2*time.Minute, 2*time.Second).Should(Equal("quay.io/kairos/core-opensuse:latest"))
+}
+
+func itHasTheCorrectLabels() {
+	Eventually(func() string {
+		b, _ := kubectl.GetData("default", "jobs", "hello-kairos", "jsonpath={.spec.template.metadata.labels.osbuild}")
+		fmt.Printf("looking for label workloadhello-kairos = %+v\n", string(b))
+		return string(b)
+	}, 2*time.Minute, 2*time.Second).Should(Equal("workloadhello-kairos"))
+}
+
+func itCopiesTheArtifacts() {
+	nginxNamespace := "osartifactbuilder-operator-system"
+	Eventually(func() string {
+		podName := strings.TrimSpace(findPodsWithLabel(nginxNamespace, "app.kubernetes.io/name=osbuilder-nginx"))
+
+		out, _ := kubectl.RunCommandWithOutput(nginxNamespace, podName, "ls /usr/share/nginx/html")
+
+		return out
+	}, 15*time.Minute, 2*time.Second).Should(MatchRegexp("hello-kairos.iso"))
+}
+
+func itCleansUpRoleBindings() {
+	nginxNamespace := "osartifactbuilder-operator-system"
+	Eventually(func() string {
+		rb := findRoleBindings(nginxNamespace)
+
+		return rb
+	}, 3*time.Minute, 2*time.Second).ShouldNot(MatchRegexp("hello-kairos"))
+}
+
+func itDeletesTheArtifacts() {
+	nginxNamespace := "osartifactbuilder-operator-system"
+	Eventually(func() string {
+		podName := findPodsWithLabel(nginxNamespace, "app.kubernetes.io/name=osbuilder-nginx")
+
+		out, err := kubectl.RunCommandWithOutput(nginxNamespace, podName, "ls	/usr/share/nginx/html")
+		Expect(err).ToNot(HaveOccurred(), out)
+
+		return out
+	}, 3*time.Minute, 2*time.Second).ShouldNot(MatchRegexp("hello-kairos.iso"))
+}
+
+func findPodsWithLabel(namespace, label string) string {
+	kubectlCommand := fmt.Sprintf("kubectl get pods -n %s -l %s --no-headers -o custom-columns=\":metadata.name\" | head -n1", namespace, label)
+	cmd := exec.Command("bash", "-c", kubectlCommand)
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	Expect(err).ToNot(HaveOccurred(), stderr.String())
+
+	return strings.TrimSpace(out.String())
+}
+
+func findRoleBindings(namespace string) string {
+	kubectlCommand := fmt.Sprintf("kubectl get rolebindings -n %s --no-headers -o custom-columns=\":metadata.name\"", namespace)
+	cmd := exec.Command("bash", "-c", kubectlCommand)
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	Expect(err).ToNot(HaveOccurred(), stderr.String())
+
+	return strings.TrimSpace(out.String())
+}

tests/fixtures/simple.yaml

@@ -6,7 +6,7 @@ spec:
   imageName: "quay.io/kairos/core-opensuse:latest"
   iso: true
   bundles:
-  - quay.io/kairos/packages:goreleaser-utils-1.11.1
+  - quay.io/kairos/packages:goreleaser-utils-1.13.1
   grubConfig: |
           search --file --set=root /boot/kernel.xz
           set default=0
@@ -49,4 +49,4 @@ spec:
               device: "/dev/sda"
               reboot: true
               poweroff: true
-              auto: true # Required, for automated installations
+              auto: true # Required, for automated installations

tools-image/Dockerfile

@@ -1,35 +1,96 @@
-ARG ELEMENTAL_CLI_VERSION=0.20220921
+# https://quay.io/repository/kairos/packages?tab=tags&tag=latest
+ARG ELEMENTAL_CLI_VERSION=0.20221121.1
 ARG LEAP_VERSION=15.4
-ARG LUET_VERSION=0.32.5
+ARG LUET_VERSION=0.33.0
-FROM quay.io/kairos/packages:elemental-cli-system-0.20220921 AS elemental
+FROM quay.io/kairos/packages:elemental-cli-system-$ELEMENTAL_CLI_VERSION AS elemental
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 
+### TODO: Replace those naked Dockerfiles copies with luet install so we can keep track of all versioning with 1 repository tag
+### 1) Add the kairos repository with a reference
+### 2) populate folders accordingly
+
 ## amd64 Live CD artifacts
-FROM quay.io/kairos/packages:grub2-livecd-0.0.4  AS grub2
+FROM quay.io/kairos/packages:grub2-livecd-0.0.4 AS grub2
 FROM quay.io/kairos/packages:grub2-efi-image-livecd-0.0.4 AS efi
 
+## RPI64
+
+## Firmware is in the amd64 repo (noarch)
+FROM quay.io/kairos/packages:u-boot-rpi64-firmware-2021.01-5.1 AS rpi-u-boot
+FROM quay.io/kairos/packages:raspberrypi-firmware-firmware-2021.03.10-2.1 AS rpi-firmware
+FROM quay.io/kairos/packages:raspberrypi-firmware-config-firmware-2021.03.10-2.1 AS rpi-firmware-config
+FROM quay.io/kairos/packages:raspberrypi-firmware-dt-firmware-2021.03.15-2.1 AS rpi-firmware-dt
+
+## PineBook64 Pro
+FROM quay.io/kairos/packages:u-boot-rockchip-arm-vendor-blob-0.1 AS pinebook-u-boot
+
+## Generic ARM artifacts
+FROM quay.io/kairos/packages-arm64:grub-efi-static-0.1  AS grub-efi
+FROM quay.io/kairos/packages-arm64:grub-config-static-0.1  AS grub-config
+FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.1   AS grub-artifacts
+
+ ## RAW images
+FROM quay.io/kairos/packages:grub-efi-static-0.1  AS grub-raw-efi
+FROM quay.io/kairos/packages:grub-config-static-0.1  AS grub-raw-config
+FROM quay.io/kairos/packages:grub-artifacts-static-0.1   AS grub-raw-artifacts
+
 FROM opensuse/leap:$LEAP_VERSION
 COPY --from=elemental /usr/bin/elemental /usr/bin/elemental
 COPY --from=luet /usr/bin/luet /usr/bin/luet
 
+# x86_64 ISOs
 COPY --from=grub2 / /grub2
 COPY --from=efi / /efi
 
+# RAW images
+COPY --from=grub-raw-efi / /raw/grub
+COPY --from=grub-raw-config / /raw/grubconfig
+COPY --from=grub-raw-artifacts / /raw/grubartifacts
+
+# RPI64
+COPY --from=rpi-u-boot / /rpi/u-boot
+COPY --from=rpi-firmware / /rpi/rpi-firmware
+COPY --from=rpi-firmware-config / /rpi/rpi-firmware-config
+COPY --from=rpi-firmware-dt / /rpi/rpi-firmware-dt
+
+# Pinebook
+COPY --from=pinebook-u-boot / /pinebookpro/u-boot
+
+# Generic
+COPY --from=grub-efi / /arm/grub/efi
+COPY --from=grub-config / /arm/grub/config
+COPY --from=grub-artifacts / /arm/grub/artifacts
+
 RUN zypper ref && zypper dup -y
+
+## ISO Build depedencies
 RUN zypper ref && zypper in -y xfsprogs parted util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2
 RUN mkdir /config
+
 # Arm image build deps
 RUN zypper in -y jq docker git curl gptfdisk kpartx sudo
 # Netboot
 RUN zypper in -y cdrtools
-# ISO build
+# cloud images
+RUN zypper in -y bc qemu-tools
+
+# ISO build config
 COPY ./config.yaml /config/manifest.yaml
 COPY ./entrypoint.sh /entrypoint.sh
 COPY ./add-cloud-init.sh /add-cloud-init.sh
 
-# ARM
+# ARM helpers
 COPY ./build-arm-image.sh /build-arm-image.sh
 COPY ./arm /arm
+COPY ./prepare_arm_images.sh /prepare_arm_images.sh
+
+# RAW images helpers
+COPY ./gce.sh /gce.sh
+COPY ./raw-images.sh /raw-images.sh
+COPY ./azure.sh /azure.sh
+COPY ./netboot.sh /netboot.sh
+
+COPY defaults.yaml /defaults.yaml
 
 
 ENTRYPOINT [ "/entrypoint.sh" ]

tools-image/arm/boards/pinebookpro.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+LOADER_OFFSET=${LOADER_OFFSET:-"64"}
+LOADER_IMAGE=${LOADER_IMAGE:-"idbloader.img"}
+UBOOT_IMAGE=${UBOOT_IMAGE:-"u-boot.itb"}
+UBOOT_OFFSET=${UBOOT_OFFSET:-"16384"}
+
+echo "Writing idbloader"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${LOADER_IMAGE} of="$image" conv=fsync seek=${LOADER_OFFSET}
+echo "Writing u-boot image"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${UBOOT_IMAGE} of="$image" conv=fsync seek=${UBOOT_OFFSET}
+sync $image

tools-image/arm/boards/rpi64.sh

@@ -11,18 +11,14 @@ if [ -z "$image" ]; then
     exit 1
 fi
 
-if [ ! -e "$WORKDIR/luet.yaml" ]; then
-    ls -liah $WORKDIR
-    echo "No valid config file"
-    cat "$WORKDIR/luet.yaml"
-    exit 1
-fi
 set -ax
 TEMPDIR="$(mktemp -d)"
 echo $TEMPDIR
 mount "${device}p1" "${TEMPDIR}"
-sudo luet install --config $WORKDIR/luet.yaml -y --system-target $TEMPDIR firmware/u-boot-rpi64
+
-sudo luet install --config $WORKDIR/luet.yaml -y --system-target $TEMPDIR firmware/raspberrypi-firmware
+for dir in /rpi/u-boot /rpi/rpi-firmware /rpi/rpi-firmware-config /rpi/rpi-firmware-dt
-sudo luet install --config $WORKDIR/luet.yaml -y --system-target $TEMPDIR firmware/raspberrypi-firmware-config
+do
-sudo luet install --config $WORKDIR/luet.yaml -y --system-target $TEMPDIR firmware/raspberrypi-firmware-dt
+    cp -rfv ${dir}/* $TEMPDIR
+done
+
 umount "${TEMPDIR}"

tools-image/azure.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Transform a raw image disk to azure vhd
+RAWIMAGE="$1"
+VHDDISK="${2:-disk.vhd}"
+cp -rf $RAWIMAGE $VHDDISK.work
+
+MB=$((1024*1024))
+size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
+# shellcheck disable=SC2004
+ROUNDED_SIZE=$(((($size+$MB-1)/$MB)*$MB))
+echo "Resizing raw image to $ROUNDED_SIZE"
+qemu-img resize -f raw "$VHDDISK.work" $ROUNDED_SIZE
+echo "Converting $RAWIMAGE to $VHDDISK"
+qemu-img convert -f raw -o subformat=fixed,force_size -O vpc "$VHDDISK.work" "$VHDDISK"
+echo "Done"
+rm -rf "$VHDDISK.work"

tools-image/build-arm-image.sh

@@ -100,8 +100,7 @@ usage()
     echo " --local: (optional) Use local repository when building"
     echo " --directory: (optional) A directory which will be used for active/passive/recovery system"
     echo " --model: (optional) The board model"
-    echo " --final-repo: (optional) The luet repository used to download bits required for building"
+    echo " --efi-dir: (optional) A directory with files which will be added to the efi partition"
-    echo " --repo-type: (optional) The luet repository type used to download bits required for building"
     exit 1
 }
 
@@ -176,6 +175,10 @@ while [ "$#" -gt 0 ]; do
             shift 1
             model=$1
             ;;
+        --efi-dir)
+            shift 1
+            efi_dir=$1
+            ;;
         --final-repo)
             shift 1
             final_repo=$1
@@ -242,17 +245,6 @@ WORKDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
 TARGET=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
 STATEDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
 
-# Create a luet config for grabbing packages from local and remote repositories (local with high prio)
-cat << EOF > $WORKDIR/luet.yaml
-repositories:
-  - name: cOS
-    enable: true
-    urls:
-      - $final_repo
-    type: $repo_type
-    priority: 90
-EOF
-
 
 export WORKDIR
 
@@ -280,7 +272,7 @@ ensure_dir_structure $TARGET
 # Download the container image
 if [ -z "$directory" ]; then
   echo ">>> Downloading container image"
-  elemental pull-image $container_image $TARGET
+  elemental pull-image $( (( $local_build == 'true')) && printf %s '--local' ) $container_image $TARGET
 else
   echo ">>> Copying files from $directory"
   rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET
@@ -311,22 +303,9 @@ cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img
 tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
 
 # Install real grub config to recovery
-if [ -z "$manifest" ]; then
+cp -rfv /arm/grub/config/* $RECOVERY
-  luet install --config $WORKDIR/luet.yaml -y --system-target $RECOVERY system/grub2-config 
+mkdir -p $RECOVERY/grub2
-  luet install --config $WORKDIR/luet.yaml -y --system-target $RECOVERY/grub2 system/grub2-artifacts
+cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
-else
-  while IFS=$'\t' read -r name target ; do
-    if [ "$target" == "root/grub2" ]; then
-      luet install --no-spinner --system-target $RECOVERY/grub2 -y "$name"
-    fi
-    if [ "$target" == "root" ]; then
-      luet install --no-spinner --system-target $RECOVERY -y "$name"
-    fi
-  done < <("${YQ_PACKAGES_COMMAND[@]}" | jq -r ".raw_disk.$model.packages[] | [.name, .target] | @tsv")
-fi
-
- # Remove luet cache
-rm -rf $RECOVERY/var $RECOVERY/grub2/var
 
 sync
 
@@ -338,19 +317,12 @@ if [ -z "$EFI" ]; then
   exit 1
 fi
 
-if [ -z "$manifest" ]; then
+cp -rfv /arm/grub/efi/* $EFI
-  luet install --config $WORKDIR/luet.yaml -y --system-target $EFI system/grub2-efi-image
+if [ -n "$EFI" ] && [  -n "$efi_dir" ]; then
-else
+  echo "Copy $efi_dir to EFI directory"
-  while IFS=$'\t' read -r name target ; do
+  cp -rfv $efi_dir/* $EFI
-    if [ "$target" == "efi" ]; then
-      luet install --no-spinner --system-target $EFI -y "$name"
-    fi
-  done < <("${YQ_PACKAGES_COMMAND[@]}" | jq -r ".raw_disk.$model.packages[] | [.name, .target] | @tsv")
 fi
 
- # Remove luet cache
-rm -rf $EFI/var
-
 echo ">> Writing image and partition table"
 dd if=/dev/zero of="${output_image}" bs=1024000 count="${size}" || exit 1
 if [ "$model" == "rpi64" ]; then 
@@ -411,15 +383,20 @@ mount $recovery $WORKDIR/recovery
 mount $state $WORKDIR/state
 mount $efi $WORKDIR/efi
 
+mkdir $WORKDIR/persistent
+mount $persistent $WORKDIR/persistent
+mkdir -p $WORKDIR/persistent/cloud-config
+
+cp -rfv /defaults.yaml $WORKDIR/persistent/cloud-config/01_defaults.yaml
+
+grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=Kairos"
+
 # Set a OEM config file if specified
 if [ -n "$config" ]; then
   echo ">> Copying $config OEM config file"
-  mkdir $WORKDIR/persistent
-  mount $persistent $WORKDIR/persistent
-  mkdir $WORKDIR/persistent/cloud-config
   get_url $config $WORKDIR/persistent/cloud-config/99_custom.yaml
-  umount $WORKDIR/persistent
 fi
+umount $WORKDIR/persistent
 
 # Copy over content
 cp -arf $EFI/* $WORKDIR/efi

tools-image/defaults.yaml

@@ -0,0 +1,8 @@
+
+name: "Default user"
+stages:
+   initramfs:
+     - name: "Set default user/pass"
+       users:
+        kairos:
+          passwd: "kairos"

tools-image/gce.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Transform a raw image disk to gce compatible
+RAWIMAGE="$1"
+OUT="${2:-$RAWIMAGE.gce.raw}"
+cp -rf $RAWIMAGE $OUT
+
+GB=$((1024*1024*1024))
+size=$(qemu-img info -f raw --output json "$OUT" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
+# shellcheck disable=SC2004
+ROUNDED_SIZE=$(echo "$size/$GB+1"|bc)
+echo "Resizing raw image from \"$size\"MB to \"$ROUNDED_SIZE\"GB"
+qemu-img resize -f raw "$OUT" "$ROUNDED_SIZE"G
+echo "Compressing raw image $OUT to $OUT.tar.gz"
+tar -c -z --format=oldgnu -f "$OUT".tar.gz $OUT

tools-image/netboot.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Extracts squashfs, kernel, initrd and generates a ipxe template script
+
+ISO=$1
+OUTPUT_NAME=$2
+ARTIFACT_NAME=$(basename $OUTPUT_NAME)
+
+isoinfo -x /rootfs.squashfs -R -i $ISO > $OUTPUT_NAME.squashfs
+isoinfo -x /boot/kernel -R -i $ISO > $OUTPUT_NAME-kernel
+isoinfo -x /boot/initrd -R -i $ISO > $OUTPUT_NAME-initrd
+
+URL=${URL:-https://github.com/kairos-io/kairos/releases/download}
+
+cat > $OUTPUT_NAME.ipxe << EOF
+#!ipxe
+set url ${URL}/
+set kernel $ARTIFACT_NAME-kernel
+set initrd $ARTIFACT_NAME-initrd
+set rootfs $ARTIFACT_NAME.squashfs
+# set config https://example.com/machine-config
+# set cmdline extra.values=1
+kernel \${url}/\${kernel} initrd=\${initrd} ip=dhcp rd.cos.disable root=live:\${url}/\${rootfs} netboot nodepair.enable config_url=\${config} console=tty1 console=ttyS0 \${cmdline}
+initrd \${url}/\${initrd}
+boot
+EOF

tools-image/prepare_arm_images.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+# This script prepares Kairos state, recovery, oem and pesistent partitions as img files.
+
+set -e
+
+# Temp dir used during build
+WORKDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
+TARGET=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
+STATEDIR=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
+
+: "${OEM_LABEL:=COS_OEM}"
+: "${RECOVERY_LABEL:=COS_RECOVERY}"
+: "${ACTIVE_LABEL:=COS_ACTIVE}"
+: "${PASSIVE_LABEL:=COS_PASSIVE}"
+: "${PERSISTENT_LABEL:=COS_PERSISTENT}"
+: "${SYSTEM_LABEL:=COS_SYSTEM}"
+: "${STATE_LABEL:=COS_STATE}"
+  
+size="${SIZE:-7544}"
+state_size="${STATE_SIZE:-4992}"
+recovery_size="${RECOVERY_SIZE:-2192}"
+default_active_size="${DEFAULT_ACTIVE_SIZE:-2400}"
+
+container_image="${container_image:-quay.io/kairos/kairos-opensuse-leap-arm-rpi:v1.5.1-k3sv1.25.6-k3s1}"
+
+ensure_dir_structure() {
+    local target=$1
+    for mnt in /sys /proc /dev /tmp /boot /usr/local /oem
+    do
+        if [ ! -d "${target}${mnt}" ]; then
+          mkdir -p ${target}${mnt}
+        fi
+    done
+}
+
+mkdir -p ${STATEDIR}/cOS
+
+dd if=/dev/zero of=${STATEDIR}/cOS/active.img bs=1M count=$default_active_size
+
+mkfs.ext2 ${STATEDIR}/cOS/active.img -L ${ACTIVE_LABEL}
+
+
+LOOP=$(losetup --show -f ${STATEDIR}/cOS/active.img)
+if [ -z "$LOOP" ]; then
+  echo "No device"
+  exit 1
+fi
+
+mount -t ext2 $LOOP $TARGET
+
+ensure_dir_structure $TARGET
+
+# Download the container image
+if [ -z "$directory" ]; then
+  echo ">>> Downloading container image"
+  luet util unpack $container_image $TARGET
+else
+  echo ">>> Copying files from $directory"
+  rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET
+fi
+
+umount $TARGET
+sync
+
+losetup -d $LOOP
+
+
+echo ">> Preparing passive.img"
+cp -rfv ${STATEDIR}/cOS/active.img ${STATEDIR}/cOS/passive.img
+tune2fs -L ${PASSIVE_LABEL} ${STATEDIR}/cOS/passive.img
+
+
+# Preparing recovery
+echo ">> Preparing recovery.img"
+RECOVERY=$(mktemp -d --tmpdir arm-builder.XXXXXXXXXX)
+mkdir -p ${RECOVERY}/cOS
+cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img
+tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
+
+# Install real grub config to recovery
+cp -rfv /arm/grub/config/* $RECOVERY
+mkdir -p $RECOVERY/grub2
+cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
+
+dd if=/dev/zero of=recovery_partition.img bs=1M count=$recovery_size
+dd if=/dev/zero of=state_partition.img bs=1M count=$state_size
+
+mkfs.ext4 -F -L ${RECOVERY_LABEL} recovery_partition.img
+LOOP=$(losetup --show -f recovery_partition.img)
+mkdir -p $WORKDIR/recovery
+mount $LOOP $WORKDIR/recovery
+cp -arf $RECOVERY/* $WORKDIR/recovery
+umount $WORKDIR/recovery
+losetup -d $LOOP
+
+mkfs.ext4 -F -L ${STATE_LABEL} state_partition.img
+LOOP=$(losetup --show -f state_partition.img)
+mkdir -p $WORKDIR/state
+mount $LOOP $WORKDIR/state
+cp -arf $STATEDIR/* $WORKDIR/state
+grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=Kairos"
+umount $WORKDIR/state
+losetup -d $LOOP
+
+cp -rfv state_partition.img bootloader/
+cp -rfv recovery_partition.img bootloader/
+
+## Optional, prepare COS_OEM and COS_PERSISTENT
+
+# Create the grubenv forcing first boot to be on recovery system
+mkdir -p $WORKDIR/oem
+cp -rfv /defaults.yaml $WORKDIR/oem/01_defaults.yaml
+
+# Create a 64MB filesystem for OEM volume
+truncate -s $((64*1024*1024)) bootloader/oem.img
+mkfs.ext2 -L "${OEM_LABEL}" -d $WORKDIR/oem bootloader/oem.img
+
+# Create a 2GB filesystem for COS_PERSISTENT volume
+truncate -s $((2048*1024*1024)) bootloader/persistent.img
+mkfs.ext2 -L "${PERSISTENT_LABEL}" bootloader/persistent.img

tools-image/raw-images.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# Generates EFI bootable images (statically)
+# This is a re-adaptation of https://github.com/rancher/elemental-toolkit/blob/v0.8.10-1/images/img-builder.sh, which was dropped
+# How to use:
+# First extract the image which you want to create an image from:
+### luet util unpack <image> rootfs
+# Then convert it to a raw disk (EFI only):
+### docker run -v $PWD:/output --entrypoint /raw-images.sh -ti --rm test-image /output/rootfs /output/foo.raw cloud-init.yaml
+
+: "${OEM_LABEL:=COS_OEM}"
+: "${RECOVERY_LABEL:=COS_RECOVERY}"
+: "${EXTEND:=}"
+
+DIRECTORY=$1
+OUT=${2:-disk.raw}
+CONFIG=$3
+
+echo "Output: $OUT"
+
+set -e
+
+mkdir -p /build/root/grub2
+mkdir /build/root/cOS
+mkdir /build/efi
+
+cp -rf /raw/grub/* /build/efi
+cp -rf /raw/grubconfig/* /build/root
+cp -rf /raw/grubartifacts/* /build/root/grub2
+
+echo "Generating squashfs from $DIRECTORY"
+mksquashfs $DIRECTORY recovery.squashfs -b 1024k -comp xz -Xbcj x86
+mv recovery.squashfs /build/root/cOS/recovery.squashfs
+
+grub2-editenv /build/root/grub_oem_env set "default_menu_entry=Kairos"
+
+# Create a 2GB filesystem for RECOVERY including the contents for root (grub config and squasfs container)
+truncate -s $((2048*1024*1024)) rootfs.part
+mkfs.ext2 -L "${RECOVERY_LABEL}" -d /build/root rootfs.part
+
+# Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg
+truncate -s $((20*1024*1024)) efi.part
+
+mkfs.fat -F16 -n COS_GRUB efi.part
+mcopy -s -i efi.part /build/efi/EFI ::EFI
+
+# Create the grubenv forcing first boot to be on recovery system
+mkdir -p /build/oem
+cp /build/root/etc/cos/grubenv_firstboot /build/oem/grubenv
+if [ -n "$CONFIG" ]; then
+  echo "Copying config file ($CONFIG)"
+  cp $CONFIG /build/oem
+fi
+
+# Create a 64MB filesystem for OEM volume
+truncate -s $((64*1024*1024)) oem.part
+mkfs.ext2 -L "${OEM_LABEL}" -d /build/oem oem.part
+
+echo "Generating image $OUT"
+# Create disk image, add 3MB of initial free space to disk, 1MB is for proper alignement, 2MB are for the hybrid legacy boot.
+truncate -s $((3*1024*1024)) $OUT
+{
+    cat efi.part
+    cat oem.part
+    cat rootfs.part
+} >> $OUT
+
+# Add an extra MB at the end of the disk for the gpt headers, in fact 34 sectors would be enough, but adding some more does not hurt.
+truncate -s "+$((1024*1024))" $OUT
+
+if [ -n "$EXTEND" ]; then
+  echo "Extending image of $EXTEND MB"
+  truncate -s "+$(($EXTEND*1024*1024))" $OUT
+fi
+
+# Create the partition table in $OUT (assumes sectors of 512 bytes)
+sgdisk -n 1:2048:+2M -c 1:legacy -t 1:EF02 $OUT
+sgdisk -n 2:0:+20M -c 2:UEFI -t 2:EF00 $OUT
+sgdisk -n 3:0:+64M -c 3:oem -t 3:8300 $OUT
+sgdisk -n 4:0:+2048M -c 4:root -t 4:8300 $OUT