Update build-arm-image.sh

Signed-off-by: Itxaka <itxaka.garcia@spectrocloud.com>

.github/workflows/bump_repos.yml

@@ -0,0 +1,26 @@
+name: Bump repositories
+on:
+  schedule:
+    - cron: 0 20 * * *
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install earthly
+        uses: Luet-lab/luet-install-action@v1
+        with:
+          repository: quay.io/kairos/packages
+          packages: utils/earthly
+      - name: Bump cos 🔧
+        run: earthly +bump-repositories
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+          push-to-fork: ci-forks/osbuilder
+          commit-message: ':arrow_up: Update repositories'
+          title: ':arrow_up: Update repositories'
+          body: Bump of Kairos repositories
+          signoff: true

.github/workflows/image.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Prepare
         id: prep

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Test
         run: |
           make kind-e2e-tests

.github/workflows/tool-image.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Prepare
         id: prep

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.18 as builder
+FROM golang:1.20 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Earthfile

@@ -0,0 +1,22 @@
+VERSION 0.6
+
+last-commit-packages:
+    FROM quay.io/skopeo/stable
+    RUN dnf install -y jq
+    WORKDIR build
+    RUN skopeo list-tags docker://quay.io/kairos/packages | jq -rc '.Tags | map(select( (. | contains("-repository.yaml")) )) | sort_by(. | sub("v";"") | sub("-repository.yaml";"") | sub("-";"") | split(".") | map(tonumber) ) | .[-1]' > REPO_AMD64
+    RUN skopeo list-tags docker://quay.io/kairos/packages-arm64 | jq -rc '.Tags | map(select( (. | contains("-repository.yaml")) )) | sort_by(. | sub("v";"") | sub("-repository.yaml";"") | sub("-";"") | split(".") | map(tonumber) ) | .[-1]' > REPO_ARM64
+    SAVE ARTIFACT REPO_AMD64 REPO_AMD64
+    SAVE ARTIFACT REPO_ARM64 REPO_ARM64
+
+bump-repositories:
+    FROM mikefarah/yq
+    WORKDIR build
+    COPY +last-commit-packages/REPO_AMD64 REPO_AMD64
+    COPY +last-commit-packages/REPO_ARM64 REPO_ARM64
+    ARG REPO_AMD64=$(cat REPO_AMD64)
+    ARG REPO_ARM64=$(cat REPO_ARM64)
+    COPY tools-image/luet.yaml luet.yaml
+    RUN yq eval ".repositories[0] |= . * { \"reference\": \"${REPO_AMD64}\" }" -i luet.yaml
+    RUN yq eval ".repositories[1] |= . * { \"reference\": \"${REPO_ARM64}\" }" -i luet.yaml
+    SAVE ARTIFACT luet.yaml AS LOCAL tools-image/luet.yaml

api/v1alpha2/groupversion_info.go

@@ -14,10 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package v1alpha1 contains API Schema definitions for the build v1alpha1 API group
+// Package v1alpha2 contains API Schema definitions for the build v1alpha2 API group
 // +kubebuilder:object:generate=true
 // +groupName=build.kairos.io
-package v1alpha1
+package v1alpha2
 
 import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -26,7 +26,7 @@ import (
 
 var (
 	// GroupVersion is group version used to register these objects
-	GroupVersion = schema.GroupVersion{Group: "build.kairos.io", Version: "v1alpha1"}
+	GroupVersion = schema.GroupVersion{Group: "build.kairos.io", Version: "v1alpha2"}
 
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
 	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}

api/v1alpha2/osartifact_types.go

@@ -14,23 +14,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package v1alpha1
+package v1alpha2
 
 import (
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
 // OSArtifactSpec defines the desired state of OSArtifact
 type OSArtifactSpec struct {
-	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-
-	// Foo is an example field of OSArtifact. Edit osartifact_types.go to remove/update
 	ImageName string `json:"imageName,omitempty"`
-	// This needs to be revisited
+
 	ISO bool `json:"iso,omitempty"`
 
 	//Disk-only stuff
@@ -42,47 +37,42 @@ type OSArtifactSpec struct {
 	Netboot    bool   `json:"netboot,omitempty"`
 	NetbootURL string `json:"netbootURL,omitempty"`
 
-	// TODO: treat cloudconfig as a secret, and take a secretRef where to store it (optionally)
-	CloudConfig string `json:"cloudConfig,omitempty"`
-	GRUBConfig  string `json:"grubConfig,omitempty"`
+	CloudConfigRef *SecretKeySelector `json:"cloudConfigRef,omitempty"`
+	GRUBConfig     string             `json:"grubConfig,omitempty"`
 
-	Bundles     []string `json:"bundles,omitempty"`
-	PullOptions Pull     `json:"pull,omitempty"`
-	OSRelease   string   `json:"osRelease,omitempty"`
-	// TODO: Currently not used. Reserved to be used when we have a way to push to registries.
-	PushOptions Push `json:"push,omitempty"`
-}
+	Bundles   []string `json:"bundles,omitempty"`
+	OSRelease string   `json:"osRelease,omitempty"`
 
-type Push struct {
-	Push                         bool               `json:"push,omitempty"`
-	ImageName                    string             `json:"imageName,omitempty"`
-	ContainerRegistryCredentials *SecretKeySelector `json:"containerRegistryCredentials,omitempty"`
-}
-
-type Pull struct {
-	ContainerRegistryCredentials *SecretKeySelector `json:"containerRegistryCredentials,omitempty"`
-}
-type LocalObjectReference struct {
-	Name string `json:"name"`
+	ImagePullSecrets []corev1.LocalObjectReference     `json:"imagePullSecrets,omitempty"`
+	Exporters        []batchv1.JobSpec                 `json:"exporters,omitempty"`
+	Volume           *corev1.PersistentVolumeClaimSpec `json:"volume,omitempty"`
 }
 
 type SecretKeySelector struct {
-	LocalObjectReference `json:",inline"`
-	// +optional
-	Namespace string `json:"namespace,omitempty"`
+	Name string `json:"name"`
 	// +optional
 	Key string `json:"key,omitempty"`
 }
 
+type ArtifactPhase string
+
+const (
+	Pending   = "Pending"
+	Building  = "Building"
+	Exporting = "Exporting"
+	Ready     = "Ready"
+	Error     = "Error"
+)
+
 // OSArtifactStatus defines the observed state of OSArtifact
 type OSArtifactStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-	Phase string `json:"phase,omitempty"`
+	// +kubebuilder:default=Pending
+	Phase ArtifactPhase `json:"phase,omitempty"`
 }
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
 
 // OSArtifact is the Schema for the osartifacts API
 type OSArtifact struct {

api/v1alpha2/zz_generated.deepcopy.go

@@ -19,27 +19,14 @@ limitations under the License.
 
 // Code generated by controller-gen. DO NOT EDIT.
 
-package v1alpha1
+package v1alpha2
 
 import (
+	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *LocalObjectReference) DeepCopyInto(out *LocalObjectReference) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalObjectReference.
-func (in *LocalObjectReference) DeepCopy() *LocalObjectReference {
-	if in == nil {
-		return nil
-	}
-	out := new(LocalObjectReference)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OSArtifact) DeepCopyInto(out *OSArtifact) {
 	*out = *in
@@ -102,13 +89,33 @@ func (in *OSArtifactList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) {
 	*out = *in
+	if in.CloudConfigRef != nil {
+		in, out := &in.CloudConfigRef, &out.CloudConfigRef
+		*out = new(SecretKeySelector)
+		**out = **in
+	}
 	if in.Bundles != nil {
 		in, out := &in.Bundles, &out.Bundles
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
-	in.PullOptions.DeepCopyInto(&out.PullOptions)
-	in.PushOptions.DeepCopyInto(&out.PushOptions)
+	if in.ImagePullSecrets != nil {
+		in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
+		*out = make([]v1.LocalObjectReference, len(*in))
+		copy(*out, *in)
+	}
+	if in.Exporters != nil {
+		in, out := &in.Exporters, &out.Exporters
+		*out = make([]batchv1.JobSpec, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Volume != nil {
+		in, out := &in.Volume, &out.Volume
+		*out = new(v1.PersistentVolumeClaimSpec)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OSArtifactSpec.
@@ -136,50 +143,9 @@ func (in *OSArtifactStatus) DeepCopy() *OSArtifactStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Pull) DeepCopyInto(out *Pull) {
-	*out = *in
-	if in.ContainerRegistryCredentials != nil {
-		in, out := &in.ContainerRegistryCredentials, &out.ContainerRegistryCredentials
-		*out = new(SecretKeySelector)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pull.
-func (in *Pull) DeepCopy() *Pull {
-	if in == nil {
-		return nil
-	}
-	out := new(Pull)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Push) DeepCopyInto(out *Push) {
-	*out = *in
-	if in.ContainerRegistryCredentials != nil {
-		in, out := &in.ContainerRegistryCredentials, &out.ContainerRegistryCredentials
-		*out = new(SecretKeySelector)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Push.
-func (in *Push) DeepCopy() *Push {
-	if in == nil {
-		return nil
-	}
-	out := new(Push)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretKeySelector) DeepCopyInto(out *SecretKeySelector) {
 	*out = *in
-	out.LocalObjectReference = in.LocalObjectReference
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeySelector.

config/rbac/kustomization.yaml

@@ -16,6 +16,3 @@ resources:
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
-
-patchesStrategicMerge:
-- role_custom.yaml

config/rbac/role.yaml

@@ -5,6 +5,49 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - build.kairos.io
   resources:

config/rbac/role_custom.yaml

@@ -1,94 +0,0 @@
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  name: manager-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - "rbac.authorization.k8s.io"
-  resources:
-  - roles
-  - rolebindings
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - build.kairos.io
-  resources:
-  - osartifacts
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - build.kairos.io
-  resources:
-  - osartifacts/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - build.kairos.io
-  resources:
-  - osartifacts/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - build.kairos.io
-  resources:
-  - osartifacts/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-  - update
-- apiGroups:
-  - "batch"
-  resources:
-  - jobs
-  verbs:
-  - get
-  - create
-  - update
-# Temporary so that it can grant these permissions to the created role
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - pods/exec
-  verbs:
-  - create

controllers/configmap.go

@@ -17,22 +17,19 @@ limitations under the License.
 package controllers
 
 import (
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
 	v1 "k8s.io/api/core/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (r *OSArtifactReconciler) genConfigMap(artifact buildv1alpha1.OSArtifact) *v1.ConfigMap {
+func (r *OSArtifactReconciler) genConfigMap(artifact *osbuilder.OSArtifact) *v1.ConfigMap {
 	return &v1.ConfigMap{
-
 		ObjectMeta: metav1.ObjectMeta{
-			Name:            artifact.Name,
-			Namespace:       artifact.Namespace,
-			OwnerReferences: genOwner(artifact),
+			Name:      artifact.Name,
+			Namespace: artifact.Namespace,
 		},
 		Data: map[string]string{
-			"config":     artifact.Spec.CloudConfig,
 			"grub.cfg":   artifact.Spec.GRUBConfig,
 			"os-release": artifact.Spec.OSRelease,
 		}}

controllers/job.go

@@ -17,35 +17,17 @@ limitations under the License.
 package controllers
 
 import (
-	"bytes"
-	"context"
 	"fmt"
-
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	"github.com/pkg/errors"
-	batchv1 "k8s.io/api/batch/v1"
-	v1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/tools/remotecommand"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
+	corev1 "k8s.io/api/core/v1"
 )
 
-func genJobLabel(s string) map[string]string {
-	return map[string]string{
-		"osbuild": "workload" + s,
-	}
-}
-
-// TODO: Handle registry auth
-// TODO: This shells out, but needs ENV_VAR with key refs mapping
-// TODO: Cache downloaded images?
-func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1alpha1.Pull) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+func unpackContainer(id, containerImage, pullImage string) corev1.Container {
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            fmt.Sprintf("pull-image-%s", id),
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -56,7 +38,7 @@ func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1al
 				"/rootfs",
 			),
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "rootfs",
 				MountPath: "/rootfs",
@@ -65,19 +47,19 @@ func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1al
 	}
 }
 
-func pushImageName(artifact buildv1alpha1.OSArtifact) string {
-	pushName := artifact.Spec.PushOptions.ImageName
+func pushImageName(artifact *osbuilder.OSArtifact) string {
+	pushName := artifact.Spec.ImageName
 	if pushName != "" {
 		return pushName
 	}
 	return artifact.Name
 }
 
-func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifact) v1.Container {
+func createImageContainer(containerImage string, artifact *osbuilder.OSArtifact) corev1.Container {
 	imageName := pushImageName(artifact)
 
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            "create-image",
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -88,7 +70,7 @@ func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifa
 				artifact.Name,
 			),
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "rootfs",
 				MountPath: "/rootfs",
@@ -101,39 +83,16 @@ func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifa
 	}
 }
 
-func createPushToServerImageContainer(containerImage string, artifactPodInfo ArtifactPodInfo) v1.Container {
-	command := fmt.Sprintf("tar cf - -C artifacts/ . | kubectl exec -i -n %s $(kubectl  get pods -l %s -n %s --no-headers -o custom-columns=\":metadata.name\" | head -n1) -- tar xf - -C %s", artifactPodInfo.Namespace, artifactPodInfo.Label, artifactPodInfo.Namespace, artifactPodInfo.Path)
-	fmt.Printf("command = %+v\n", command)
-
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		Name:            "push-to-server",
-		Image:           containerImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args:            []string{command},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-			{
-				Name:      "artifacts",
-				MountPath: "/artifacts",
-			},
-		},
-	}
-}
-
-func osReleaseContainer(containerImage string) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+func osReleaseContainer(containerImage string) corev1.Container {
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            "os-release",
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
 		Args: []string{
 			"cp -rfv /etc/os-release /rootfs/etc/os-release",
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "config",
 				MountPath: "/etc/os-release",
@@ -147,18 +106,38 @@ func osReleaseContainer(containerImage string) v1.Container {
 	}
 }
 
-func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv1.Job {
-	objMeta := genObjectMeta(artifact)
+func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *corev1.PersistentVolumeClaim {
+	if artifact.Spec.Volume == nil {
+		artifact.Spec.Volume = &corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: map[corev1.ResourceName]resource.Quantity{
+					"storage": resource.MustParse("10Gi"),
+				},
+			},
+		}
+	}
 
-	privileged := false
-	serviceAccount := true
+	pvc := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      artifact.Name + "-artifacts",
+			Namespace: artifact.Namespace,
+		},
+		Spec: *artifact.Spec.Volume,
+	}
 
+	return pvc
+}
+
+func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder.OSArtifact) *corev1.Pod {
 	cmd := fmt.Sprintf(
 		"/entrypoint.sh --debug --name %s build-iso --date=false --output /artifacts dir:/rootfs",
 		artifact.Name,
 	)
 
-	volumeMounts := []v1.VolumeMount{
+	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "artifacts",
 			MountPath: "/artifacts",
@@ -170,7 +149,7 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 	}
 
 	if artifact.Spec.GRUBConfig != "" {
-		volumeMounts = append(volumeMounts, v1.VolumeMount{
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
 			Name:      "config",
 			MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
 			SubPath:   "grub.cfg",
@@ -182,26 +161,26 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		artifact.Name,
 	)
 
-	if artifact.Spec.CloudConfig != "" {
-		volumeMounts = append(volumeMounts, v1.VolumeMount{
-			Name:      "config",
+	if artifact.Spec.CloudConfigRef != nil {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "cloudconfig",
 			MountPath: "/iso/iso-overlay/cloud_config.yaml",
-			SubPath:   "config",
+			SubPath:   artifact.Spec.CloudConfigRef.Key,
 		})
 
 		cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
 	}
 
-	if artifact.Spec.CloudConfig != "" || artifact.Spec.GRUBConfig != "" {
+	if artifact.Spec.CloudConfigRef != nil || artifact.Spec.GRUBConfig != "" {
 		cmd = fmt.Sprintf(
 			"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /artifacts dir:/rootfs",
 			artifact.Name,
 		)
 	}
 
-	buildIsoContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildIsoContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-iso",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -211,9 +190,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildCloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-cloud-image",
 		Image:           r.ToolImage,
 
@@ -225,19 +204,19 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 	}
 
 	if artifact.Spec.DiskSize != "" {
-		buildCloudImageContainer.Env = []v1.EnvVar{{
+		buildCloudImageContainer.Env = []corev1.EnvVar{{
 			Name:  "EXTEND",
 			Value: artifact.Spec.DiskSize,
 		}}
 	}
 
-	extractNetboot := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	extractNetboot := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-netboot",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
-		Env: []v1.EnvVar{{
+		Env: []corev1.EnvVar{{
 			Name:  "URL",
 			Value: artifact.Spec.NetbootURL,
 		}},
@@ -251,9 +230,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildAzureCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildAzureCloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-azure-cloud-image",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -267,9 +246,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildGCECloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildGCECloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-gce-cloud-image",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -283,251 +262,93 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	pod := v1.PodSpec{
-		AutomountServiceAccountToken: &serviceAccount,
-		ServiceAccountName:           objMeta.Name,
-		RestartPolicy:                v1.RestartPolicyNever,
-		Volumes: []v1.Volume{
+	podSpec := corev1.PodSpec{
+		AutomountServiceAccountToken: ptr(false),
+		RestartPolicy:                corev1.RestartPolicyNever,
+		Volumes: []corev1.Volume{
 			{
-				Name:         "artifacts",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+				Name: "artifacts",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: pvcName,
+						ReadOnly:  false,
+					},
+				},
 			},
 			{
 				Name:         "rootfs",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+				VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
 			},
 			{
 				Name: "config",
-				VolumeSource: v1.VolumeSource{
-					ConfigMap: &v1.ConfigMapVolumeSource{
-						LocalObjectReference: v1.LocalObjectReference{Name: artifact.Name}}},
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: artifact.Name,
+						},
+					},
+				},
 			},
 		},
 	}
 
-	pod.InitContainers = []v1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName, artifact.Spec.PullOptions)}
+	if artifact.Spec.CloudConfigRef != nil {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
+			Name: "cloudconfig",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: artifact.Spec.CloudConfigRef.Name,
+					Optional:   ptr(true),
+				},
+			},
+		})
+	}
+
+	for i := range artifact.Spec.ImagePullSecrets {
+		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, artifact.Spec.ImagePullSecrets[i])
+	}
+
+	podSpec.InitContainers = []corev1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName)}
 
 	for i, bundle := range artifact.Spec.Bundles {
-		pod.InitContainers = append(pod.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle, artifact.Spec.PullOptions))
+		podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle))
 	}
 
 	if artifact.Spec.OSRelease != "" {
-		pod.InitContainers = append(pod.InitContainers, osReleaseContainer(r.ToolImage))
+		podSpec.InitContainers = append(podSpec.InitContainers, osReleaseContainer(r.ToolImage))
 	}
 
 	if artifact.Spec.ISO || artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
+		podSpec.Containers = append(podSpec.Containers, buildIsoContainer)
 	}
 
 	if artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, extractNetboot)
+		podSpec.Containers = append(podSpec.Containers, extractNetboot)
 	}
 
-	if artifact.Spec.CloudImage || artifact.Spec.AzureImage || artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildCloudImageContainer)
+	if artifact.Spec.CloudImage {
+		podSpec.Containers = append(podSpec.Containers, buildCloudImageContainer)
 	}
 
 	if artifact.Spec.AzureImage {
-		pod.InitContainers = append(pod.InitContainers, buildAzureCloudImageContainer)
+		podSpec.Containers = append(podSpec.Containers, buildAzureCloudImageContainer)
 	}
 
 	if artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildGCECloudImageContainer)
+		podSpec.Containers = append(podSpec.Containers, buildGCECloudImageContainer)
 	}
 
-	pod.InitContainers = append(pod.InitContainers, createImageContainer(r.ToolImage, artifact))
+	podSpec.Containers = append(podSpec.Containers, createImageContainer(r.ToolImage, artifact))
 
-	pod.Containers = []v1.Container{
-		createPushToServerImageContainer(r.CopierImage, r.ArtifactPodInfo),
-	}
-
-	jobLabels := genJobLabel(artifact.Name)
-
-	job := batchv1.Job{
-		ObjectMeta: objMeta,
-		Spec: batchv1.JobSpec{
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: jobLabels,
-				},
-				Spec: pod,
-			},
-		},
-	}
-
-	return &job
-}
-
-// createServiceAccount creates a service account that has the permissions to
-// copy the artifacts to the http server Pod. This service account is used for
-// the "push to server" container.
-func (r *OSArtifactReconciler) createCopierServiceAccount(ctx context.Context, objMeta metav1.ObjectMeta) error {
-	sa, err := r.clientSet.CoreV1().
-		ServiceAccounts(objMeta.Namespace).Get(ctx, objMeta.Name, metav1.GetOptions{})
-	if sa == nil || apierrors.IsNotFound(err) {
-		t := true
-		_, err := r.clientSet.CoreV1().ServiceAccounts(objMeta.Namespace).Create(ctx,
-			&v1.ServiceAccount{
-				ObjectMeta:                   objMeta,
-				AutomountServiceAccountToken: &t,
-			}, metav1.CreateOptions{})
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
-}
-
-// func (r *OSArtifactReconciler) createCopierRole(ctx context.Context, objMeta metav1.ObjectMeta) error {
-// 	role, err := r.clientSet.RbacV1().
-// 		Roles(objMeta.Namespace).
-// 		Get(ctx, objMeta.Name, metav1.GetOptions{})
-// 	if role == nil || apierrors.IsNotFound(err) {
-// 		_, err := r.clientSet.RbacV1().Roles(objMeta.Namespace).Create(ctx,
-// 			&rbacv1.Role{
-// 				ObjectMeta: objMeta,
-// 				Rules: []rbacv1.PolicyRule{
-// 					// TODO: The actual permissions we need is that to copy to a Pod.
-// 					// The Pod is on another namespace, so we need a cluster wide permission.
-// 					// This can get viral because the controller needs to have the permissions
-// 					// if it is to grant them to the Job.
-// 					{
-// 						Verbs:     []string{"list"},
-// 						APIGroups: []string{""},
-// 						Resources: []string{"pods"},
-// 					},
-// 				},
-// 			},
-// 			metav1.CreateOptions{},
-// 		)
-// 		if err != nil {
-// 			return err
-// 		}
-// 	}
-
-// 	return err
-// }
-
-func (r *OSArtifactReconciler) createCopierRoleBinding(ctx context.Context, objMeta metav1.ObjectMeta) error {
-	newrb := &rbacv1.RoleBinding{
+	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      objMeta.Name,
-			Namespace: r.ArtifactPodInfo.Namespace,
-			// TODO: We can't have cross-namespace owners. The role binding will have to deleted explicitly by the reconciler (finalizer?)
-			// OwnerReferences: objMeta.OwnerReferences,
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "Role",
-			Name:     r.ArtifactPodInfo.Role,
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      objMeta.Name,
-				Namespace: objMeta.Namespace,
-			},
+			GenerateName: artifact.Name + "-",
+			Namespace:    artifact.Namespace,
 		},
+		Spec: podSpec,
 	}
-
-	rb, err := r.clientSet.RbacV1().
-		RoleBindings(r.ArtifactPodInfo.Namespace).
-		Get(ctx, objMeta.Name, metav1.GetOptions{})
-	if rb == nil || apierrors.IsNotFound(err) {
-		_, err := r.clientSet.RbacV1().
-			RoleBindings(r.ArtifactPodInfo.Namespace).
-			Create(ctx, newrb, metav1.CreateOptions{})
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
 }
 
-// createRBAC creates a ServiceAccount, and a binding to the CopierRole so that
-// the container that copies the artifacts to the http server Pod has the
-// permissions to do so.
-func (r *OSArtifactReconciler) createRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	objMeta := genObjectMeta(artifact)
-
-	err := r.createCopierServiceAccount(ctx, objMeta)
-	if err != nil {
-		return errors.Wrap(err, "creating a service account")
-	}
-
-	err = r.createCopierRoleBinding(ctx, objMeta)
-	if err != nil {
-		return errors.Wrap(err, "creating a role binding for the copy-role")
-	}
-
-	return err
-}
-
-// removeRBAC deletes the role binding between the service account of this artifact
-// and the CopierRole. The ServiceAccount is removed automatically through the Owner
-// relationship with the OSArtifact. The RoleBinding can't have it as an owner
-// because it is in a different Namespace.
-func (r *OSArtifactReconciler) removeRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	err := r.clientSet.RbacV1().RoleBindings(r.ArtifactPodInfo.Namespace).
-		Delete(ctx, artifact.Name, metav1.DeleteOptions{})
-	// Ignore not found. No need to do anything.
-	if err != nil && apierrors.IsNotFound(err) {
-		return nil
-	}
-
-	return err
-}
-
-func (r *OSArtifactReconciler) removeArtifacts(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	//Finding Pods using labels
-	fmt.Printf("r.ArtifactPodInfo = %+v\n", r.ArtifactPodInfo.Label)
-	pods, err := r.clientSet.CoreV1().Pods(r.ArtifactPodInfo.Namespace).
-		List(ctx, metav1.ListOptions{LabelSelector: r.ArtifactPodInfo.Label})
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("listing pods with label %s in namespace %s", r.ArtifactPodInfo.Label, r.ArtifactPodInfo.Namespace))
-	}
-	if len(pods.Items) < 1 {
-		return errors.New("No artifact pod found")
-	}
-	pod := pods.Items[0]
-
-	stdout, stderr, err := r.executeRemoteCommand(r.ArtifactPodInfo.Namespace, pod.Name, fmt.Sprintf("rm -rf %s/%s.*", r.ArtifactPodInfo.Path, artifact.Name))
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("%s\n%s", stdout, stderr))
-	}
-	return nil
-}
-
-func (r *OSArtifactReconciler) executeRemoteCommand(namespace, podName, command string) (string, string, error) {
-	buf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
-	request := r.clientSet.CoreV1().RESTClient().
-		Post().
-		Namespace(namespace).
-		Resource("pods").
-		Name(podName).
-		SubResource("exec").
-		VersionedParams(&v1.PodExecOptions{
-			Command: []string{"/bin/sh", "-c", command},
-			Stdin:   false,
-			Stdout:  true,
-			Stderr:  true,
-			TTY:     true,
-		}, scheme.ParameterCodec)
-
-	exec, err := remotecommand.NewSPDYExecutor(r.restConfig, "POST", request.URL())
-	if err != nil {
-		return "", "", err
-	}
-	err = exec.Stream(remotecommand.StreamOptions{
-		Stdout: buf,
-		Stderr: errBuf,
-	})
-	if err != nil {
-		return "", "", fmt.Errorf("%w Failed executing command %s on %v/%v", err, command, namespace, podName)
-	}
-
-	return buf.String(), errBuf.String(), nil
+func ptr[T any](val T) *T {
+	return &val
 }

controllers/osartifact_controller.go

@@ -19,230 +19,286 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"time"
-
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	"github.com/pkg/errors"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"sigs.k8s.io/cluster-api/util/patch"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
-const FinalizerName = "build.kairos.io/osbuilder-finalizer"
-
-type ArtifactPodInfo struct {
-	Label     string
-	Namespace string
-	Path      string
-	Role      string
-}
+const (
+	FinalizerName                   = "build.kairos.io/osbuilder-finalizer"
+	artifactLabel                   = "build.kairos.io/artifact"
+	artifactExporterIndexAnnotation = "build.kairos.io/export-index"
+)
 
 // OSArtifactReconciler reconciles a OSArtifact object
 type OSArtifactReconciler struct {
 	client.Client
-	Scheme                               *runtime.Scheme
-	restConfig                           *rest.Config
-	clientSet                            *kubernetes.Clientset
 	ServingImage, ToolImage, CopierImage string
-	ArtifactPodInfo                      ArtifactPodInfo
 }
 
-func genObjectMeta(artifact buildv1alpha1.OSArtifact) metav1.ObjectMeta {
-	return metav1.ObjectMeta{
-		Name:            artifact.Name,
-		Namespace:       artifact.Namespace,
-		OwnerReferences: genOwner(artifact),
-	}
-}
+func (r *OSArtifactReconciler) InjectClient(c client.Client) error {
+	r.Client = c
 
-func genOwner(artifact buildv1alpha1.OSArtifact) []metav1.OwnerReference {
-	return []metav1.OwnerReference{
-		*metav1.NewControllerRef(&artifact.ObjectMeta, schema.GroupVersionKind{
-			Group:   buildv1alpha1.GroupVersion.Group,
-			Version: buildv1alpha1.GroupVersion.Version,
-			Kind:    "OSArtifact",
-		}),
-	}
+	return nil
 }
 
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/finalizers,verbs=update
+//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
+//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;delete;watch
+//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;
+//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;create;
+//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete
 
-// TODO: Is this ^ how I should have created rbac permissions for the controller?
-//       - git commit all changes
-//       - generate code with kubebuilder
-//       - check if my permissions were removed
-//       - do it properly
-
-// Reconcile is part of the main kubernetes reconciliation loop which aims to
-// move the current state of the cluster closer to the desired state.
-// TODO(user): Modify the Reconcile function to compare the state specified by
-// the OSArtifact object against the actual cluster state, and then
-// perform operations to make the cluster state reflect the state specified by
-// the user.
-//
-// For more details, check Reconcile and its Result here:
-// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile
 func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 
-	var osbuild buildv1alpha1.OSArtifact
-	if err := r.Get(ctx, req.NamespacedName, &osbuild); err != nil {
+	var artifact osbuilder.OSArtifact
+	if err := r.Get(ctx, req.NamespacedName, &artifact); err != nil {
 		if apierrors.IsNotFound(err) {
 			return ctrl.Result{}, nil
 		}
-		return ctrl.Result{}, err
+		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Reconciling %v", osbuild))
-
-	stop, err := r.handleFinalizer(ctx, &osbuild)
-	if err != nil || stop {
-		return ctrl.Result{}, err
+	if artifact.DeletionTimestamp != nil {
+		controllerutil.RemoveFinalizer(&artifact, FinalizerName)
+		return ctrl.Result{}, r.Update(ctx, &artifact)
 	}
 
-	// generate configmap required for building a custom image
-	desiredConfigMap := r.genConfigMap(osbuild)
-	logger.Info(fmt.Sprintf("Checking configmap %v", osbuild))
-
-	cfgMap, err := r.clientSet.CoreV1().ConfigMaps(req.Namespace).Get(ctx, desiredConfigMap.Name, metav1.GetOptions{})
-	if cfgMap == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating config map %v", desiredConfigMap))
-
-		_, err = r.clientSet.CoreV1().ConfigMaps(req.Namespace).Create(ctx, desiredConfigMap, metav1.CreateOptions{})
-		if err != nil {
-			logger.Error(err, "Failed while creating config map")
-			return ctrl.Result{}, err
+	if !controllerutil.ContainsFinalizer(&artifact, FinalizerName) {
+		controllerutil.AddFinalizer(&artifact, FinalizerName)
+		if err := r.Update(ctx, &artifact); err != nil {
+			return ctrl.Result{Requeue: true}, err
 		}
+	}
+
+	logger.Info(fmt.Sprintf("Reconciling %s/%s", artifact.Namespace, artifact.Name))
+
+	switch artifact.Status.Phase {
+	case osbuilder.Exporting:
+		return r.checkExport(ctx, &artifact)
+	case osbuilder.Ready, osbuilder.Error:
+		return ctrl.Result{}, nil
+	default:
+		return r.checkBuild(ctx, &artifact)
+	}
+}
+
+func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	// generate configmap required for building a custom image
+	cm := r.genConfigMap(artifact)
+	if cm.Labels == nil {
+		cm.Labels = map[string]string{}
+	}
+	cm.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, cm, r.Scheme()); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
-	if err != nil {
+	if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Checking deployment %v", osbuild))
-
-	err = r.createRBAC(ctx, osbuild)
-	if err != nil {
+	pvc := r.newArtifactPVC(artifact)
+	if pvc.Labels == nil {
+		pvc.Labels = map[string]string{}
+	}
+	pvc.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, pvc, r.Scheme()); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+	if err := r.Create(ctx, pvc); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	desiredJob := r.genJob(osbuild)
-	job, err := r.clientSet.BatchV1().Jobs(req.Namespace).Get(ctx, desiredJob.Name, metav1.GetOptions{})
-	if job == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating Job %v", job))
+	pod := r.newBuilderPod(pvc.Name, artifact)
+	if pod.Labels == nil {
+		pod.Labels = map[string]string{}
+	}
+	pod.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, pod, r.Scheme()); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+	if err := r.Create(ctx, pod); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
 
-		_, err = r.clientSet.BatchV1().Jobs(req.Namespace).Create(ctx, desiredJob, metav1.CreateOptions{})
-		if err != nil {
-			logger.Error(err, "Failed while creating job")
+	artifact.Status.Phase = osbuilder.Building
+	if err := r.Status().Update(ctx, artifact); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	var pods corev1.PodList
+	if err := r.List(ctx, &pods, &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labels.Set{
+			artifactLabel: artifact.Name,
+		}),
+	}); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	for _, pod := range pods.Items {
+		switch pod.Status.Phase {
+		case corev1.PodSucceeded:
+			artifact.Status.Phase = osbuilder.Exporting
+			return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
+		case corev1.PodFailed:
+			artifact.Status.Phase = osbuilder.Error
+			return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
+		case corev1.PodPending, corev1.PodRunning:
 			return ctrl.Result{}, nil
 		}
-
-		return ctrl.Result{Requeue: true}, nil
 	}
-	if err != nil {
+
+	return r.startBuild(ctx, artifact)
+}
+
+func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	var jobs batchv1.JobList
+	if err := r.List(ctx, &jobs, &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labels.Set{
+			artifactLabel: artifact.Name,
+		}),
+	}); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Updating state %v", osbuild))
-
-	copy := osbuild.DeepCopy()
-
-	helper, err := patch.NewHelper(&osbuild, r.Client)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-	if job.Status.Succeeded > 0 {
-		copy.Status.Phase = "Ready"
-	} else if copy.Status.Phase != "Building" {
-		copy.Status.Phase = "Building"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
-	defer cancel()
-	if err := helper.Patch(ctx, copy); err != nil {
-		return ctrl.Result{}, errors.Wrapf(err, "couldn't patch osbuild %q", copy.Name)
+	indexedJobs := make(map[string]*batchv1.Job, len(artifact.Spec.Exporters))
+	for _, job := range jobs.Items {
+		if job.GetAnnotations() != nil {
+			if idx, ok := job.GetAnnotations()[artifactExporterIndexAnnotation]; ok {
+				indexedJobs[idx] = &job
+			}
+		}
 	}
 
-	// for _, c := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
-	// 	if c.State.Terminated != nil && c.State.Terminated.ExitCode != 0 {
-	// 		packageBuildCopy.Status.State = "Failed"
-	// 	}
-	// }
+	var pvcs corev1.PersistentVolumeClaimList
+	var pvc *corev1.PersistentVolumeClaim
+	if err := r.List(ctx, &pvcs, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{artifactLabel: artifact.Name})}); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	for _, item := range pvcs.Items {
+		pvc = &item
+		break
+	}
+
+	if pvc == nil {
+		log.FromContext(ctx).Error(nil, "failed to locate pvc for artifact, this should not happen")
+		return ctrl.Result{}, fmt.Errorf("failed to locate artifact pvc")
+	}
+
+	var succeeded int
+	for i := range artifact.Spec.Exporters {
+		idx := fmt.Sprintf("%d", i)
+
+		job := indexedJobs[idx]
+		if job == nil {
+			job = &batchv1.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-export-%s", artifact.Name, idx),
+					Namespace: artifact.Namespace,
+					Annotations: map[string]string{
+						artifactExporterIndexAnnotation: idx,
+					},
+					Labels: map[string]string{
+						artifactLabel: artifact.Name,
+					},
+				},
+				Spec: artifact.Spec.Exporters[i],
+			}
+
+			job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
+				Name: "artifacts",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: pvc.Name,
+						ReadOnly:  true,
+					},
+				},
+			})
+
+			if err := controllerutil.SetOwnerReference(artifact, job, r.Scheme()); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+
+			if err := r.Create(ctx, job); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+
+		} else if job.Spec.Completions == nil || *job.Spec.Completions == 1 {
+			if job.Status.Succeeded > 0 {
+				succeeded++
+			}
+		} else if *job.Spec.BackoffLimit <= job.Status.Failed {
+			artifact.Status.Phase = osbuilder.Error
+			if err := r.Status().Update(ctx, artifact); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+			break
+		}
+	}
+
+	if succeeded == len(artifact.Spec.Exporters) {
+		artifact.Status.Phase = osbuilder.Ready
+		if err := r.Status().Update(ctx, artifact); err != nil {
+			return ctrl.Result{Requeue: true}, err
+		}
+	}
 
 	return ctrl.Result{}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *OSArtifactReconciler) SetupWithManager(mgr ctrl.Manager) error {
-
-	cfg := mgr.GetConfig()
-	clientset, err := kubernetes.NewForConfig(cfg)
-	if err != nil {
-		return err
-	}
-	r.restConfig = cfg
-	r.clientSet = clientset
-
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&buildv1alpha1.OSArtifact{}).
+		For(&osbuilder.OSArtifact{}).
+		Owns(&osbuilder.OSArtifact{}).
+		Watches(
+			&source.Kind{Type: &corev1.Pod{}},
+			handler.EnqueueRequestsFromMapFunc(r.findOwningArtifact),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+		).
+		Watches(
+			&source.Kind{Type: &batchv1.Job{}},
+			handler.EnqueueRequestsFromMapFunc(r.findOwningArtifact),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+		).
 		Complete(r)
 }
 
-// Returns true if reconciliation should stop or false otherwise
-func (r *OSArtifactReconciler) handleFinalizer(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) (bool, error) {
-	// examine DeletionTimestamp to determine if object is under deletion
-	if osbuild.DeletionTimestamp.IsZero() {
-		// The object is not being deleted, so if it does not have our finalizer,
-		// then lets add the finalizer and update the object. This is equivalent
-		// registering our finalizer.
-		if !controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
-			controllerutil.AddFinalizer(osbuild, FinalizerName)
-			if err := r.Update(ctx, osbuild); err != nil {
-				return true, err
-			}
-		}
-	} else {
-		// The object is being deleted
-		if controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
-			// our finalizer is present, so lets handle any external dependency
-			if err := r.finalize(ctx, osbuild); err != nil {
-				// if fail to delete the external dependency here, return with error
-				// so that it can be retried
-				return true, err
-			}
-
-			// remove our finalizer from the list and update it.
-			controllerutil.RemoveFinalizer(osbuild, FinalizerName)
-			if err := r.Update(ctx, osbuild); err != nil {
-				return true, err
-			}
-		}
-
-		// Stop reconciliation as the item is being deleted
-		return true, nil
+func (r *OSArtifactReconciler) findOwningArtifact(obj client.Object) []reconcile.Request {
+	if obj.GetLabels() == nil {
+		return nil
 	}
 
-	return false, nil
-}
-
-// - Remove artifacts from the server Pod
-// - Delete role-binding (because it doesn't have the OSArtifact as an owner and won't be deleted automatically)
-func (r *OSArtifactReconciler) finalize(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) error {
-	if err := r.removeRBAC(ctx, *osbuild); err != nil {
-		return err
-	}
-
-	if err := r.removeArtifacts(ctx, *osbuild); err != nil {
-		return err
+	if artifactName, ok := obj.GetLabels()[artifactLabel]; ok {
+		return []reconcile.Request{
+			{
+				NamespacedName: types.NamespacedName{
+					Name:      artifactName,
+					Namespace: obj.GetNamespace(),
+				},
+			},
+		}
 	}
 
 	return nil

go.mod

@@ -4,8 +4,8 @@ go 1.18
 
 require (
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.1.3
-	github.com/onsi/gomega v1.18.1
+	github.com/onsi/ginkgo/v2 v2.9.5
+	github.com/onsi/gomega v1.27.7
 	github.com/pkg/errors v0.9.1
 	github.com/rancher-sandbox/ele-testhelpers v0.0.0-20220614101555-2eddf3b113e2
 	k8s.io/api v0.24.0
@@ -33,18 +33,20 @@ require (
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobuffalo/flect v0.2.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -65,19 +67,20 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.24.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect

go.sum

@@ -179,6 +179,8 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -191,8 +193,9 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I=
 github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -236,6 +239,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -258,6 +263,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
@@ -280,7 +287,6 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -409,12 +415,18 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042
 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
+github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -654,6 +666,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -751,9 +765,13 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -764,6 +782,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -831,8 +851,9 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y=
 golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -970,6 +991,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1000,6 +1023,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

main.go

@@ -31,7 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
+	buildv1alpha2 "github.com/kairos-io/osbuilder/api/v1alpha2"
 	"github.com/kairos-io/osbuilder/controllers"
 	//+kubebuilder:scaffold:imports
 )
@@ -44,7 +44,7 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
-	utilruntime.Must(buildv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(buildv1alpha2.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 
@@ -53,21 +53,12 @@ func main() {
 	var enableLeaderElection bool
 	var probeAddr string
 	var serveImage, toolImage, copierImage string
-	var copyToPodLabel, copyToNamespace, copyToPath, copierRole string
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 
-	flag.StringVar(&copierImage, "copier-image", "quay.io/kairos/kubectl", "The image that is used to copy artifacts to the server pod.")
-	flag.StringVar(&serveImage, "serve-image", "nginx", "Serve image.")
 	// It needs luet inside
 	flag.StringVar(&toolImage, "tool-image", "quay.io/kairos/osbuilder-tools:latest", "Tool image.")
 
-	// Information on where to copy the artifacts
-	flag.StringVar(&copyToPodLabel, "copy-to-pod-label", "", "The label of the Pod to which artifacts should be copied.")
-	flag.StringVar(&copyToNamespace, "copy-to-namespace", "", "The namespace of the copy-to-pod-label Pod.")
-	flag.StringVar(&copyToPath, "copy-to-path", "", "The path under which to copy artifacts in the copy-to-pod-label Pod.")
-	flag.StringVar(&copierRole, "copy-role", "", "The name or the Kubernetes Role that has the permissions to copy artifacts to the copy-to-pod-label Pod")
-
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -105,17 +96,9 @@ func main() {
 	}
 
 	if err = (&controllers.OSArtifactReconciler{
-		Client:       mgr.GetClient(),
 		ServingImage: serveImage,
 		ToolImage:    toolImage,
 		CopierImage:  copierImage,
-		ArtifactPodInfo: controllers.ArtifactPodInfo{
-			Label:     copyToPodLabel,
-			Namespace: copyToNamespace,
-			Path:      copyToPath,
-			Role:      copierRole,
-		},
-		Scheme: mgr.GetScheme(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "OSArtifact")
 		os.Exit(1)

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

tests/e2e/e2e_simple_test.go

@@ -1,114 +1,157 @@
 package e2e_test
 
 import (
-	"bytes"
-	"fmt"
-	"os/exec"
-	"strings"
-	"time"
-
+	"context"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	kubectl "github.com/rancher-sandbox/ele-testhelpers/kubectl"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/dynamic"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"time"
 )
 
 var _ = Describe("ISO build test", func() {
-	//k := kubectl.New()
-	Context("registration", func() {
+	k8s := dynamic.NewForConfigOrDie(ctrl.GetConfigOrDie())
+	scheme := runtime.NewScheme()
+	_ = osbuilder.AddToScheme(scheme)
 
-		AfterEach(func() {
-			kubectl.New().Delete("osartifacts", "-n", "default", "hello-kairos")
+	var artifactName string
+	artifacts := k8s.Resource(schema.GroupVersionResource{Group: osbuilder.GroupVersion.Group, Version: osbuilder.GroupVersion.Version, Resource: "osartifacts"}).Namespace("default")
+	pods := k8s.Resource(schema.GroupVersionResource{Group: corev1.GroupName, Version: corev1.SchemeGroupVersion.Version, Resource: "pods"}).Namespace("default")
+	pvcs := k8s.Resource(schema.GroupVersionResource{Group: corev1.GroupName, Version: corev1.SchemeGroupVersion.Version, Resource: "persistentvolumeclaims"}).Namespace("default")
+	jobs := k8s.Resource(schema.GroupVersionResource{Group: batchv1.GroupName, Version: batchv1.SchemeGroupVersion.Version, Resource: "jobs"}).Namespace("default")
+
+	artifact := &osbuilder.OSArtifact{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "OSArtifact",
+			APIVersion: osbuilder.GroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "simple-",
+		},
+		Spec: osbuilder.OSArtifactSpec{
+			ImageName: "quay.io/kairos/core-opensuse:latest",
+			ISO:       true,
+			DiskSize:  "",
+			Exporters: []batchv1.JobSpec{
+				{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							RestartPolicy: corev1.RestartPolicyNever,
+							Containers: []corev1.Container{
+								{
+									Name:    "test",
+									Image:   "debian:latest",
+									Command: []string{"bash"},
+									Args:    []string{"-xec", "[ -f /artifacts/*.iso ]"},
+									VolumeMounts: []corev1.VolumeMount{
+										{
+											Name:      "artifacts",
+											ReadOnly:  true,
+											MountPath: "/artifacts",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	uArtifact := unstructured.Unstructured{}
+	uArtifact.Object, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(artifact)
+	resp, err := artifacts.Create(context.TODO(), &uArtifact, metav1.CreateOptions{})
+	Expect(err).ToNot(HaveOccurred())
+	artifactName = resp.GetName()
+
+	Context("simple", func() {
+		artifactLabelSelectorReq, _ := labels.NewRequirement("build.kairos.io/artifact", selection.Equals, []string{artifactName})
+		artifactLabelSelector := labels.NewSelector().Add(*artifactLabelSelectorReq)
+
+		It("starts the build", func() {
+			Eventually(func(g Gomega) {
+				w, err := pods.Watch(context.TODO(), metav1.ListOptions{LabelSelector: artifactLabelSelector.String()})
+				Expect(err).ToNot(HaveOccurred())
+
+				var stopped bool
+				for !stopped {
+					event, ok := <-w.ResultChan()
+
+					stopped = event.Type != watch.Deleted && event.Type != watch.Error || !ok
+				}
+			}).WithTimeout(time.Hour).Should(Succeed())
 		})
 
-		It("creates a simple iso", func() {
-			err := kubectl.Apply("", "../../tests/fixtures/simple.yaml")
+		It("exports the artifacts", func() {
+			Eventually(func(g Gomega) {
+				w, err := jobs.Watch(context.TODO(), metav1.ListOptions{LabelSelector: artifactLabelSelector.String()})
+				Expect(err).ToNot(HaveOccurred())
+
+				var stopped bool
+				for !stopped {
+					event, ok := <-w.ResultChan()
+
+					stopped = event.Type != watch.Deleted && event.Type != watch.Error || !ok
+				}
+			}).WithTimeout(time.Hour).Should(Succeed())
+		})
+
+		It("artifact successfully builds", func() {
+			Eventually(func(g Gomega) {
+				w, err := artifacts.Watch(context.TODO(), metav1.ListOptions{})
+				Expect(err).ToNot(HaveOccurred())
+
+				var artifact osbuilder.OSArtifact
+				var stopped bool
+				for !stopped {
+					event, ok := <-w.ResultChan()
+					stopped = !ok
+
+					if event.Type == watch.Modified && event.Object.(*unstructured.Unstructured).GetName() == artifactName {
+						err := scheme.Convert(event.Object, &artifact, nil)
+						Expect(err).ToNot(HaveOccurred())
+						stopped = artifact.Status.Phase == osbuilder.Ready
+					}
+
+				}
+			}).WithTimeout(time.Hour).Should(Succeed())
+		})
+
+		It("cleans up resources on deleted", func() {
+			err := artifacts.Delete(context.TODO(), artifactName, metav1.DeleteOptions{})
 			Expect(err).ToNot(HaveOccurred())
 
-			itHasTheCorrectImage()
-			itHasTheCorrectLabels()
-			itCopiesTheArtifacts()
-
-			By("deleting the custom resource", func() {
-				err = kubectl.New().Delete("osartifacts", "-n", "default", "hello-kairos")
+			Eventually(func(g Gomega) int {
+				res, err := artifacts.List(context.TODO(), metav1.ListOptions{})
 				Expect(err).ToNot(HaveOccurred())
-			})
-
-			itCleansUpRoleBindings()
-			itDeletesTheArtifacts()
+				return len(res.Items)
+			}).WithTimeout(time.Minute).Should(Equal(0))
+			Eventually(func(g Gomega) int {
+				res, err := pods.List(context.TODO(), metav1.ListOptions{LabelSelector: artifactLabelSelector.String()})
+				Expect(err).ToNot(HaveOccurred())
+				return len(res.Items)
+			}).WithTimeout(time.Minute).Should(Equal(0))
+			Eventually(func(g Gomega) int {
+				res, err := pvcs.List(context.TODO(), metav1.ListOptions{LabelSelector: artifactLabelSelector.String()})
+				Expect(err).ToNot(HaveOccurred())
+				return len(res.Items)
+			}).WithTimeout(time.Minute).Should(Equal(0))
+			Eventually(func(g Gomega) int {
+				res, err := jobs.List(context.TODO(), metav1.ListOptions{LabelSelector: artifactLabelSelector.String()})
+				Expect(err).ToNot(HaveOccurred())
+				return len(res.Items)
+			}).WithTimeout(time.Minute).Should(Equal(0))
 		})
 	})
 })
-
-func itHasTheCorrectImage() {
-	Eventually(func() string {
-		b, _ := kubectl.GetData("default", "osartifacts", "hello-kairos", "jsonpath={.spec.imageName}")
-		fmt.Printf("looking for image core-opensuse:latest = %+v\n", string(b))
-		return string(b)
-	}, 2*time.Minute, 2*time.Second).Should(Equal("quay.io/kairos/core-opensuse:latest"))
-}
-
-func itHasTheCorrectLabels() {
-	Eventually(func() string {
-		b, _ := kubectl.GetData("default", "jobs", "hello-kairos", "jsonpath={.spec.template.metadata.labels.osbuild}")
-		fmt.Printf("looking for label workloadhello-kairos = %+v\n", string(b))
-		return string(b)
-	}, 2*time.Minute, 2*time.Second).Should(Equal("workloadhello-kairos"))
-}
-
-func itCopiesTheArtifacts() {
-	nginxNamespace := "osartifactbuilder-operator-system"
-	Eventually(func() string {
-		podName := strings.TrimSpace(findPodsWithLabel(nginxNamespace, "app.kubernetes.io/name=osbuilder-nginx"))
-
-		out, _ := kubectl.RunCommandWithOutput(nginxNamespace, podName, "ls /usr/share/nginx/html")
-
-		return out
-	}, 15*time.Minute, 2*time.Second).Should(MatchRegexp("hello-kairos.iso"))
-}
-
-func itCleansUpRoleBindings() {
-	nginxNamespace := "osartifactbuilder-operator-system"
-	Eventually(func() string {
-		rb := findRoleBindings(nginxNamespace)
-
-		return rb
-	}, 3*time.Minute, 2*time.Second).ShouldNot(MatchRegexp("hello-kairos"))
-}
-
-func itDeletesTheArtifacts() {
-	nginxNamespace := "osartifactbuilder-operator-system"
-	Eventually(func() string {
-		podName := findPodsWithLabel(nginxNamespace, "app.kubernetes.io/name=osbuilder-nginx")
-
-		out, err := kubectl.RunCommandWithOutput(nginxNamespace, podName, "ls	/usr/share/nginx/html")
-		Expect(err).ToNot(HaveOccurred(), out)
-
-		return out
-	}, 3*time.Minute, 2*time.Second).ShouldNot(MatchRegexp("hello-kairos.iso"))
-}
-
-func findPodsWithLabel(namespace, label string) string {
-	kubectlCommand := fmt.Sprintf("kubectl get pods -n %s -l %s --no-headers -o custom-columns=\":metadata.name\" | head -n1", namespace, label)
-	cmd := exec.Command("bash", "-c", kubectlCommand)
-	var out bytes.Buffer
-	var stderr bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	Expect(err).ToNot(HaveOccurred(), stderr.String())
-
-	return strings.TrimSpace(out.String())
-}
-
-func findRoleBindings(namespace string) string {
-	kubectlCommand := fmt.Sprintf("kubectl get rolebindings -n %s --no-headers -o custom-columns=\":metadata.name\"", namespace)
-	cmd := exec.Command("bash", "-c", kubectlCommand)
-	var out bytes.Buffer
-	var stderr bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	Expect(err).ToNot(HaveOccurred(), stderr.String())
-
-	return strings.TrimSpace(out.String())
-}

tools-image/Dockerfile

@@ -1,17 +1,24 @@
 # https://quay.io/repository/kairos/packages?tab=tags&tag=latest
-ARG ELEMENTAL_CLI_VERSION=0.20221121.1
 ARG LEAP_VERSION=15.4
 ARG LUET_VERSION=0.33.0
-FROM quay.io/kairos/packages:elemental-cli-system-$ELEMENTAL_CLI_VERSION AS elemental
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 
-### TODO: Replace those naked Dockerfiles copies with luet install so we can keep track of all versioning with 1 repository tag
-### 1) Add the kairos repository with a reference
-### 2) populate folders accordingly
+FROM opensuse/leap:$LEAP_VERSION as luet-install
+COPY --from=luet /usr/bin/luet /usr/bin/luet
+ENV LUET_NOLOCK=true
+ENV TMPDIR=/tmp
+COPY luet.yaml /etc/luet/luet.yaml
+RUN luet install -y system/elemental-cli
+RUN luet install -y livecd/grub2 --system-target /grub2
+RUN luet install -y livecd/grub2-efi-image --system-target /efi
+
+# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
+RUN rm -Rf /grub2/var
+RUN rm -Rf /efi/var
 
 ## amd64 Live CD artifacts
-FROM quay.io/kairos/packages:grub2-livecd-0.0.4 AS grub2
-FROM quay.io/kairos/packages:grub2-efi-image-livecd-0.0.4 AS efi
+FROM quay.io/kairos/packages:grub2-livecd-0.0.6 AS grub2
+FROM quay.io/kairos/packages:grub2-efi-image-livecd-0.0.6 AS efi
 
 ## RPI64
 
@@ -25,9 +32,9 @@ FROM quay.io/kairos/packages:raspberrypi-firmware-dt-firmware-2021.03.15-2.1 AS
 FROM quay.io/kairos/packages:u-boot-rockchip-arm-vendor-blob-0.1 AS pinebook-u-boot
 
 ## Generic ARM artifacts
-FROM quay.io/kairos/packages-arm64:grub-efi-static-0.1  AS grub-efi
-FROM quay.io/kairos/packages-arm64:grub-config-static-0.1  AS grub-config
-FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.1   AS grub-artifacts
+FROM quay.io/kairos/packages-arm64:grub-efi-static-0.2  AS grub-efi
+FROM quay.io/kairos/packages-arm64:grub-config-static-0.3  AS grub-config
+FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.2   AS grub-artifacts
 
  ## RAW images
 FROM quay.io/kairos/packages:grub-efi-static-0.1  AS grub-raw-efi
@@ -35,12 +42,12 @@ FROM quay.io/kairos/packages:grub-config-static-0.1  AS grub-raw-config
 FROM quay.io/kairos/packages:grub-artifacts-static-0.1   AS grub-raw-artifacts
 
 FROM opensuse/leap:$LEAP_VERSION
-COPY --from=elemental /usr/bin/elemental /usr/bin/elemental
+COPY --from=luet-install /usr/bin/elemental /usr/bin/elemental
 COPY --from=luet /usr/bin/luet /usr/bin/luet
 
-# x86_64 ISOs
-COPY --from=grub2 / /grub2
-COPY --from=efi / /efi
+# ISO files
+COPY --from=luet-install /grub2 /grub2
+COPY --from=luet-install /efi /efi
 
 # RAW images
 COPY --from=grub-raw-efi / /raw/grub

tools-image/build-arm-image.sh

@@ -7,14 +7,16 @@ set -ex
 load_vars() {
 
   model=${MODEL:-odroid_c2}
-
+  disable_lvm=${DISABLE_LVM:-false}
   directory=${DIRECTORY:-}
   output_image="${OUTPUT_IMAGE:-arm.img}"
   # Img creation options. Size is in MB for all of the vars below
-  size="${SIZE:-7544}"
+  size="${SIZE:-7608}"
   state_size="${STATE_SIZE:-4992}"
+  oem_size="${OEM_SIZE:-64}"
   recovery_size="${RECOVERY_SIZE:-2192}"
   default_active_size="${DEFAULT_ACTIVE_SIZE:-2400}"
+  menu_entry="${DEFAULT_MENU_ENTRY:-Kairos}"
 
   ## Repositories
   final_repo="${FINAL_REPO:-quay.io/costoolkit/releases-teal-arm64}"
@@ -69,7 +71,15 @@ cleanup() {
   if [ -n "$oem" ]; then
     umount $oem || true
   fi
-  losetup -D || true
+
+  if [ "$disable_lvm" == "false" ]; then
+    lvremove --yes KairosVG || true
+  fi
+  MAPPER_LOOP=$(basename "$LOOP")
+    for LOOPPART in $(ls /dev/mapper/"${MAPPER_LOOP}"*| awk -F'/' {'print $4'}); do
+    dmsetup remove "${LOOPPART}" || true;
+  done;
+  losetup -d "${LOOP}" || true;
 }
 
 ensure_dir_structure() {
@@ -101,6 +111,8 @@ usage()
     echo " --directory: (optional) A directory which will be used for active/passive/recovery system"
     echo " --model: (optional) The board model"
     echo " --efi-dir: (optional) A directory with files which will be added to the efi partition"
+    echo " --disable-lvm: (optional- no arguments) LVM for the recovery and oem partitions will be disabled"
+    echo " --use-lvm: (deprecated and ignored. Kept for backwards compatibility)"
     exit 1
 }
 
@@ -187,6 +199,12 @@ while [ "$#" -gt 0 ]; do
             shift 1
             repo_type=$1
             ;;
+        --disable-lvm)
+            disable_lvm=true
+            ;;
+        --use-lvm)
+            disable_lvm=false
+            ;;
         -h)
             usage
             ;;
@@ -211,7 +229,7 @@ else
     container_image=${CONTAINER_IMAGE:-quay.io/costoolkit/examples:odroid-c2-latest}
 fi
 
-if [ -n "$cos_config"] && [ -e "$cos_config" ]; then
+if [ -n "$cos_config" ] && [ -e "$cos_config" ]; then
   source "$cos_config"
 fi
 
@@ -278,6 +296,12 @@ else
   rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET
 fi
 
+# We copy the grubmenu.cfg to a temporary location to be copied later in the state partition
+# https://github.com/kairos-io/kairos/blob/62c67e3e61d49435c362014522e5c6696335376f/overlay/files/system/oem/08_grub.yaml#L105
+# This is a hack and we need a better way: https://github.com/kairos-io/kairos/issues/1427
+tmpgrubconfig=$(mktemp /tmp/grubmeny.cfg.XXXXXX)
+cp -rfv $TARGET/etc/kairos/branding/grubmenu.cfg "${tmpgrubconfig}"
+
 umount $TARGET
 sync
 
@@ -304,8 +328,9 @@ tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
 
 # Install real grub config to recovery
 cp -rfv /arm/grub/config/* $RECOVERY
-mkdir -p $RECOVERY/grub2
+mkdir -p $RECOVERY/grub2/fonts
 cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
+mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts
 
 sync
 
@@ -323,6 +348,8 @@ if [ -n "$EFI" ] && [  -n "$efi_dir" ]; then
   cp -rfv $efi_dir/* $EFI
 fi
 
+partprobe
+
 echo ">> Writing image and partition table"
 dd if=/dev/zero of="${output_image}" bs=1024000 count="${size}" || exit 1
 if [ "$model" == "rpi64" ]; then 
@@ -331,7 +358,11 @@ else
     sgdisk -n 1:8192:+16M -c 1:EFI -t 1:0700 ${output_image}
 fi
 sgdisk -n 2:0:+${state_size}M -c 2:state -t 2:8300 ${output_image}
+if [ "$disable_lvm" == 'true' ]; then
 sgdisk -n 3:0:+${recovery_size}M -c 3:recovery -t 3:8300 ${output_image}
+else 
+sgdisk -n 3:0:+$(( ${recovery_size} + ${oem_size} ))M -c 3:lvm -t 3:8e00 ${output_image}
+fi
 sgdisk -n 4:0:+64M -c 4:persistent -t 4:8300 ${output_image}
 
 sgdisk -m 1:2:3:4 ${output_image}
@@ -366,12 +397,25 @@ efi=${device}p1
 state=${device}p2
 recovery=${device}p3
 persistent=${device}p4
+oem_lv=/dev/mapper/KairosVG-oem
+recovery_lv=/dev/mapper/KairosVG-recovery
 
 # Create partitions (RECOVERY, STATE, COS_PERSISTENT)
 mkfs.vfat -F 32 ${efi}
 fatlabel ${efi} COS_GRUB
 
+if [ "$disable_lvm" == 'true' ]; then
 mkfs.ext4 -F -L ${RECOVERY_LABEL} $recovery
+else
+pvcreate $recovery
+vgcreate KairosVG $recovery
+lvcreate -Z n -n oem -L ${oem_size} KairosVG
+lvcreate -Z n -n recovery -l 100%FREE KairosVG
+vgchange -ay
+vgmknodes
+mkfs.ext4 -F -L ${OEM_LABEL} $oem_lv
+mkfs.ext4 -F -L ${RECOVERY_LABEL} $recovery_lv
+fi
 mkfs.ext4 -F -L ${STATE_LABEL} $state
 mkfs.ext4 -F -L ${PERSISTENT_LABEL} $persistent
 
@@ -379,24 +423,38 @@ mkdir $WORKDIR/state
 mkdir $WORKDIR/recovery
 mkdir $WORKDIR/efi
 
+if [ "$disable_lvm" == 'true' ]; then
 mount $recovery $WORKDIR/recovery
+else
+mount $recovery_lv $WORKDIR/recovery
+fi
 mount $state $WORKDIR/state
 mount $efi $WORKDIR/efi
 
-mkdir $WORKDIR/persistent
-mount $persistent $WORKDIR/persistent
-mkdir -p $WORKDIR/persistent/cloud-config
 
-cp -rfv /defaults.yaml $WORKDIR/persistent/cloud-config/01_defaults.yaml
+if [ "$disable_lvm" == "false" ]; then
+  mkdir $WORKDIR/oem
+  mount $oem_lv $WORKDIR/oem
 
-grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=Kairos"
+  cp -rfv /defaults.yaml $WORKDIR/oem/01_defaults.yaml
 
-# Set a OEM config file if specified
-if [ -n "$config" ]; then
-  echo ">> Copying $config OEM config file"
-  get_url $config $WORKDIR/persistent/cloud-config/99_custom.yaml
+  # Set a OEM config file if specified
+  if [ -n "$config" ]; then
+    echo ">> Copying $config OEM config file"
+    get_url $config $WORKDIR/oem/99_custom.yaml
+  fi
+
+  umount $WORKDIR/oem
+else
+  echo "LVM disabled: Not adding default config with default user/pass and custom config file"
+  echo "Enable LVM to copy those files into /oem"
 fi
-umount $WORKDIR/persistent
+
+grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=$menu_entry"
+
+# We copy the file we saved earier to the STATE partition
+cp -rfv "${tmpgrubconfig}" $WORKDIR/state/grubmenu
+
 
 # Copy over content
 cp -arf $EFI/* $WORKDIR/efi
@@ -407,13 +465,20 @@ umount $WORKDIR/recovery
 umount $WORKDIR/state
 umount $WORKDIR/efi
 
+if [ "$disable_lvm" == 'false' ]; then
+vgchange -an
+fi
 sync
 
 # Flash uboot and vendor-specific bits
 echo ">> Performing $model specific bits.."
 /arm/boards/$model.sh ${DRIVE}
 
-kpartx -dv $DRIVE
+sync
+sleep 5
+sync
+
+kpartx -dv $DRIVE || true
 
 umount $DRIVE || true
 

tools-image/defaults.yaml

@@ -1,4 +1,4 @@
-
+#cloud-config
 name: "Default user"
 stages:
    initramfs:

tools-image/luet.yaml

@@ -0,0 +1,24 @@
+general:
+  debug: false
+  spinner_charset: 9
+logging:
+  enable_emoji: false
+repositories:
+  - name: "kairos"
+    description: "kairos repository"
+    type: "docker"
+    arch: amd64
+    cached: true
+    priority: 2
+    urls:
+      - "quay.io/kairos/packages"
+    reference: 20230512113938-repository.yaml
+  - name: "kairos-arm64"
+    description: "kairos repository arm64"
+    type: "docker"
+    arch: arm64
+    cached: true
+    priority: 2
+    urls:
+      - "quay.io/kairos/packages-arm64"
+    reference: 20230512115044-repository.yaml

tools-image/prepare_arm_images.sh

@@ -20,6 +20,7 @@ size="${SIZE:-7544}"
 state_size="${STATE_SIZE:-4992}"
 recovery_size="${RECOVERY_SIZE:-2192}"
 default_active_size="${DEFAULT_ACTIVE_SIZE:-2400}"
+menu_entry="${DEFAULT_MENU_ENTRY:-Kairos}"
 
 container_image="${container_image:-quay.io/kairos/kairos-opensuse-leap-arm-rpi:v1.5.1-k3sv1.25.6-k3s1}"
 
@@ -33,6 +34,14 @@ ensure_dir_structure() {
     done
 }
 
+mkdir -p $WORKDIR/tmpefi
+
+# Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg
+truncate -s $((20*1024*1024)) bootloader/efi.img
+cp -rfv /arm/grub/efi/* $WORKDIR/tmpefi
+mkfs.fat -F16 -n COS_GRUB bootloader/efi.img
+mcopy -s -i bootloader/efi.img $WORKDIR/tmpefi/EFI ::EFI
+
 mkdir -p ${STATEDIR}/cOS
 
 dd if=/dev/zero of=${STATEDIR}/cOS/active.img bs=1M count=$default_active_size
@@ -59,6 +68,12 @@ else
   rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET
 fi
 
+# We copy the grubmenu.cfg to a temporary location to be copied later in the state partition
+# https://github.com/kairos-io/kairos/blob/62c67e3e61d49435c362014522e5c6696335376f/overlay/files/system/oem/08_grub.yaml#L105
+# This is a hack and we need a better way: https://github.com/kairos-io/kairos/issues/1427
+tmpgrubconfig=$(mktemp /tmp/grubmeny.cfg.XXXXXX)
+cp -rfv $TARGET/etc/kairos/branding/grubmenu.cfg "${tmpgrubconfig}"
+
 umount $TARGET
 sync
 
@@ -79,8 +94,9 @@ tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
 
 # Install real grub config to recovery
 cp -rfv /arm/grub/config/* $RECOVERY
-mkdir -p $RECOVERY/grub2
+mkdir -p $RECOVERY/grub2/fonts
 cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
+mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts
 
 dd if=/dev/zero of=recovery_partition.img bs=1M count=$recovery_size
 dd if=/dev/zero of=state_partition.img bs=1M count=$state_size
@@ -98,7 +114,11 @@ LOOP=$(losetup --show -f state_partition.img)
 mkdir -p $WORKDIR/state
 mount $LOOP $WORKDIR/state
 cp -arf $STATEDIR/* $WORKDIR/state
-grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=Kairos"
+grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=$menu_entry"
+
+# We copy the file we saved earier to the STATE partition
+cp -rfv "${tmpgrubconfig}" $WORKDIR/state/grubmenu
+
 umount $WORKDIR/state
 losetup -d $LOOP
 

tools-image/raw-images.sh

@@ -10,6 +10,7 @@
 : "${OEM_LABEL:=COS_OEM}"
 : "${RECOVERY_LABEL:=COS_RECOVERY}"
 : "${EXTEND:=}"
+: "${RECOVERY_SIZE:=2048}"
 
 DIRECTORY=$1
 OUT=${2:-disk.raw}
@@ -34,7 +35,8 @@ mv recovery.squashfs /build/root/cOS/recovery.squashfs
 grub2-editenv /build/root/grub_oem_env set "default_menu_entry=Kairos"
 
 # Create a 2GB filesystem for RECOVERY including the contents for root (grub config and squasfs container)
-truncate -s $((2048*1024*1024)) rootfs.part
+# shellcheck disable=SC2004
+truncate -s $(($RECOVERY_SIZE*1024*1024)) rootfs.part
 mkfs.ext2 -L "${RECOVERY_LABEL}" -d /build/root rootfs.part
 
 # Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg
@@ -76,4 +78,4 @@ fi
 sgdisk -n 1:2048:+2M -c 1:legacy -t 1:EF02 $OUT
 sgdisk -n 2:0:+20M -c 2:UEFI -t 2:EF00 $OUT
 sgdisk -n 3:0:+64M -c 3:oem -t 3:8300 $OUT
-sgdisk -n 4:0:+2048M -c 4:root -t 4:8300 $OUT
+sgdisk -n 4:0:+${RECOVERY_SIZE}M -c 4:root -t 4:8300 $OUT