allow for custom artifact export logic

Signed-off-by: Jacob Payne <jacob@spectrocloud.com>

controllers/configmap.go

@@ -17,22 +17,19 @@ limitations under the License.
 package controllers
 
 import (
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
 	v1 "k8s.io/api/core/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func (r *OSArtifactReconciler) genConfigMap(artifact buildv1alpha1.OSArtifact) *v1.ConfigMap {
+func (r *OSArtifactReconciler) genConfigMap(artifact *osbuilder.OSArtifact) *v1.ConfigMap {
 	return &v1.ConfigMap{
-
 		ObjectMeta: metav1.ObjectMeta{
-			Name:            artifact.Name,
-			Namespace:       artifact.Namespace,
-			OwnerReferences: genOwner(artifact),
+			Name:      artifact.Name,
+			Namespace: artifact.Namespace,
 		},
 		Data: map[string]string{
-			"config":     artifact.Spec.CloudConfig,
 			"grub.cfg":   artifact.Spec.GRUBConfig,
 			"os-release": artifact.Spec.OSRelease,
 		}}

controllers/job.go

@@ -17,35 +17,17 @@ limitations under the License.
 package controllers
 
 import (
-	"bytes"
-	"context"
 	"fmt"
-
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	"github.com/pkg/errors"
-	batchv1 "k8s.io/api/batch/v1"
-	v1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/tools/remotecommand"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
+	corev1 "k8s.io/api/core/v1"
 )
 
-func genJobLabel(s string) map[string]string {
-	return map[string]string{
-		"osbuild": "workload" + s,
-	}
-}
-
-// TODO: Handle registry auth
-// TODO: This shells out, but needs ENV_VAR with key refs mapping
-// TODO: Cache downloaded images?
-func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1alpha1.Pull) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+func unpackContainer(id, containerImage, pullImage string) corev1.Container {
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            fmt.Sprintf("pull-image-%s", id),
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -56,7 +38,7 @@ func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1al
 				"/rootfs",
 			),
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "rootfs",
 				MountPath: "/rootfs",
@@ -65,19 +47,19 @@ func unpackContainer(id, containerImage, pullImage string, pullOptions buildv1al
 	}
 }
 
-func pushImageName(artifact buildv1alpha1.OSArtifact) string {
-	pushName := artifact.Spec.PushOptions.ImageName
+func pushImageName(artifact *osbuilder.OSArtifact) string {
+	pushName := artifact.Spec.ImageName
 	if pushName != "" {
 		return pushName
 	}
 	return artifact.Name
 }
 
-func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifact) v1.Container {
+func createImageContainer(containerImage string, artifact *osbuilder.OSArtifact) corev1.Container {
 	imageName := pushImageName(artifact)
 
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            "create-image",
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -88,7 +70,7 @@ func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifa
 				artifact.Name,
 			),
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "rootfs",
 				MountPath: "/rootfs",
@@ -101,39 +83,16 @@ func createImageContainer(containerImage string, artifact buildv1alpha1.OSArtifa
 	}
 }
 
-func createPushToServerImageContainer(containerImage string, artifactPodInfo ArtifactPodInfo) v1.Container {
-	command := fmt.Sprintf("tar cf - -C artifacts/ . | kubectl exec -i -n %s $(kubectl  get pods -l %s -n %s --no-headers -o custom-columns=\":metadata.name\" | head -n1) -- tar xf - -C %s", artifactPodInfo.Namespace, artifactPodInfo.Label, artifactPodInfo.Namespace, artifactPodInfo.Path)
-	fmt.Printf("command = %+v\n", command)
-
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		Name:            "push-to-server",
-		Image:           containerImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args:            []string{command},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
-			{
-				Name:      "artifacts",
-				MountPath: "/artifacts",
-			},
-		},
-	}
-}
-
-func osReleaseContainer(containerImage string) v1.Container {
-	return v1.Container{
-		ImagePullPolicy: v1.PullAlways,
+func osReleaseContainer(containerImage string) corev1.Container {
+	return corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
 		Name:            "os-release",
 		Image:           containerImage,
 		Command:         []string{"/bin/bash", "-cxe"},
 		Args: []string{
 			"cp -rfv /etc/os-release /rootfs/etc/os-release",
 		},
-		VolumeMounts: []v1.VolumeMount{
+		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:      "config",
 				MountPath: "/etc/os-release",
@@ -147,18 +106,38 @@ func osReleaseContainer(containerImage string) v1.Container {
 	}
 }
 
-func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv1.Job {
-	objMeta := genObjectMeta(artifact)
+func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *corev1.PersistentVolumeClaim {
+	if artifact.Spec.Volume == nil {
+		artifact.Spec.Volume = &corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+			Resources: corev1.ResourceRequirements{
+				Requests: map[corev1.ResourceName]resource.Quantity{
+					"storage": resource.MustParse("2Gi"),
+				},
+			},
+		}
+	}
 
-	privileged := false
-	serviceAccount := true
+	pvc := &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      artifact.Name + "-artifacts",
+			Namespace: artifact.Namespace,
+		},
+		Spec: *artifact.Spec.Volume,
+	}
 
+	return pvc
+}
+
+func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder.OSArtifact) *corev1.Pod {
 	cmd := fmt.Sprintf(
 		"/entrypoint.sh --debug --name %s build-iso --date=false --output /artifacts dir:/rootfs",
 		artifact.Name,
 	)
 
-	volumeMounts := []v1.VolumeMount{
+	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "artifacts",
 			MountPath: "/artifacts",
@@ -170,7 +149,7 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 	}
 
 	if artifact.Spec.GRUBConfig != "" {
-		volumeMounts = append(volumeMounts, v1.VolumeMount{
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
 			Name:      "config",
 			MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
 			SubPath:   "grub.cfg",
@@ -182,26 +161,26 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		artifact.Name,
 	)
 
-	if artifact.Spec.CloudConfig != "" {
-		volumeMounts = append(volumeMounts, v1.VolumeMount{
-			Name:      "config",
+	if artifact.Spec.CloudConfigRef != nil {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "cloudconfig",
 			MountPath: "/iso/iso-overlay/cloud_config.yaml",
-			SubPath:   "config",
+			SubPath:   artifact.Spec.CloudConfigRef.Key,
 		})
 
 		cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
 	}
 
-	if artifact.Spec.CloudConfig != "" || artifact.Spec.GRUBConfig != "" {
+	if artifact.Spec.CloudConfigRef != nil || artifact.Spec.GRUBConfig != "" {
 		cmd = fmt.Sprintf(
 			"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /artifacts dir:/rootfs",
 			artifact.Name,
 		)
 	}
 
-	buildIsoContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildIsoContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-iso",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -211,9 +190,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildCloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-cloud-image",
 		Image:           r.ToolImage,
 
@@ -225,19 +204,19 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 	}
 
 	if artifact.Spec.DiskSize != "" {
-		buildCloudImageContainer.Env = []v1.EnvVar{{
+		buildCloudImageContainer.Env = []corev1.EnvVar{{
 			Name:  "EXTEND",
 			Value: artifact.Spec.DiskSize,
 		}}
 	}
 
-	extractNetboot := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	extractNetboot := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-netboot",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
-		Env: []v1.EnvVar{{
+		Env: []corev1.EnvVar{{
 			Name:  "URL",
 			Value: artifact.Spec.NetbootURL,
 		}},
@@ -251,9 +230,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildAzureCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildAzureCloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-azure-cloud-image",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -267,9 +246,9 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	buildGCECloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
+	buildGCECloudImageContainer := corev1.Container{
+		ImagePullPolicy: corev1.PullAlways,
+		SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
 		Name:            "build-gce-cloud-image",
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
@@ -283,251 +262,93 @@ func (r *OSArtifactReconciler) genJob(artifact buildv1alpha1.OSArtifact) *batchv
 		VolumeMounts: volumeMounts,
 	}
 
-	pod := v1.PodSpec{
-		AutomountServiceAccountToken: &serviceAccount,
-		ServiceAccountName:           objMeta.Name,
-		RestartPolicy:                v1.RestartPolicyNever,
-		Volumes: []v1.Volume{
+	podSpec := corev1.PodSpec{
+		AutomountServiceAccountToken: ptr(false),
+		RestartPolicy:                corev1.RestartPolicyNever,
+		Volumes: []corev1.Volume{
 			{
-				Name:         "artifacts",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+				Name: "artifacts",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: pvcName,
+						ReadOnly:  false,
+					},
+				},
 			},
 			{
 				Name:         "rootfs",
-				VolumeSource: v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}},
+				VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
 			},
 			{
 				Name: "config",
-				VolumeSource: v1.VolumeSource{
-					ConfigMap: &v1.ConfigMapVolumeSource{
-						LocalObjectReference: v1.LocalObjectReference{Name: artifact.Name}}},
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: artifact.Name,
+						},
+					},
+				},
 			},
 		},
 	}
 
-	pod.InitContainers = []v1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName, artifact.Spec.PullOptions)}
+	if artifact.Spec.CloudConfigRef != nil {
+		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
+			Name: "cloudconfig",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: artifact.Spec.CloudConfigRef.Name,
+					Optional:   ptr(true),
+				},
+			},
+		})
+	}
+
+	for i := range artifact.Spec.ImagePullSecrets {
+		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, artifact.Spec.ImagePullSecrets[i])
+	}
+
+	podSpec.InitContainers = []corev1.Container{unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName)}
 
 	for i, bundle := range artifact.Spec.Bundles {
-		pod.InitContainers = append(pod.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle, artifact.Spec.PullOptions))
+		podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle))
 	}
 
 	if artifact.Spec.OSRelease != "" {
-		pod.InitContainers = append(pod.InitContainers, osReleaseContainer(r.ToolImage))
+		podSpec.InitContainers = append(podSpec.InitContainers, osReleaseContainer(r.ToolImage))
 	}
 
 	if artifact.Spec.ISO || artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
+		podSpec.Containers = append(podSpec.Containers, buildIsoContainer)
 	}
 
 	if artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, extractNetboot)
+		podSpec.Containers = append(podSpec.Containers, extractNetboot)
 	}
 
-	if artifact.Spec.CloudImage || artifact.Spec.AzureImage || artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildCloudImageContainer)
+	if artifact.Spec.CloudImage {
+		podSpec.Containers = append(podSpec.Containers, buildCloudImageContainer)
 	}
 
 	if artifact.Spec.AzureImage {
-		pod.InitContainers = append(pod.InitContainers, buildAzureCloudImageContainer)
+		podSpec.Containers = append(podSpec.Containers, buildAzureCloudImageContainer)
 	}
 
 	if artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildGCECloudImageContainer)
+		podSpec.Containers = append(podSpec.Containers, buildGCECloudImageContainer)
 	}
 
-	pod.InitContainers = append(pod.InitContainers, createImageContainer(r.ToolImage, artifact))
+	podSpec.Containers = append(podSpec.Containers, createImageContainer(r.ToolImage, artifact))
 
-	pod.Containers = []v1.Container{
-		createPushToServerImageContainer(r.CopierImage, r.ArtifactPodInfo),
-	}
-
-	jobLabels := genJobLabel(artifact.Name)
-
-	job := batchv1.Job{
-		ObjectMeta: objMeta,
-		Spec: batchv1.JobSpec{
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: jobLabels,
-				},
-				Spec: pod,
-			},
-		},
-	}
-
-	return &job
-}
-
-// createServiceAccount creates a service account that has the permissions to
-// copy the artifacts to the http server Pod. This service account is used for
-// the "push to server" container.
-func (r *OSArtifactReconciler) createCopierServiceAccount(ctx context.Context, objMeta metav1.ObjectMeta) error {
-	sa, err := r.clientSet.CoreV1().
-		ServiceAccounts(objMeta.Namespace).Get(ctx, objMeta.Name, metav1.GetOptions{})
-	if sa == nil || apierrors.IsNotFound(err) {
-		t := true
-		_, err := r.clientSet.CoreV1().ServiceAccounts(objMeta.Namespace).Create(ctx,
-			&v1.ServiceAccount{
-				ObjectMeta:                   objMeta,
-				AutomountServiceAccountToken: &t,
-			}, metav1.CreateOptions{})
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
-}
-
-// func (r *OSArtifactReconciler) createCopierRole(ctx context.Context, objMeta metav1.ObjectMeta) error {
-// 	role, err := r.clientSet.RbacV1().
-// 		Roles(objMeta.Namespace).
-// 		Get(ctx, objMeta.Name, metav1.GetOptions{})
-// 	if role == nil || apierrors.IsNotFound(err) {
-// 		_, err := r.clientSet.RbacV1().Roles(objMeta.Namespace).Create(ctx,
-// 			&rbacv1.Role{
-// 				ObjectMeta: objMeta,
-// 				Rules: []rbacv1.PolicyRule{
-// 					// TODO: The actual permissions we need is that to copy to a Pod.
-// 					// The Pod is on another namespace, so we need a cluster wide permission.
-// 					// This can get viral because the controller needs to have the permissions
-// 					// if it is to grant them to the Job.
-// 					{
-// 						Verbs:     []string{"list"},
-// 						APIGroups: []string{""},
-// 						Resources: []string{"pods"},
-// 					},
-// 				},
-// 			},
-// 			metav1.CreateOptions{},
-// 		)
-// 		if err != nil {
-// 			return err
-// 		}
-// 	}
-
-// 	return err
-// }
-
-func (r *OSArtifactReconciler) createCopierRoleBinding(ctx context.Context, objMeta metav1.ObjectMeta) error {
-	newrb := &rbacv1.RoleBinding{
+	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      objMeta.Name,
-			Namespace: r.ArtifactPodInfo.Namespace,
-			// TODO: We can't have cross-namespace owners. The role binding will have to deleted explicitly by the reconciler (finalizer?)
-			// OwnerReferences: objMeta.OwnerReferences,
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "Role",
-			Name:     r.ArtifactPodInfo.Role,
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      objMeta.Name,
-				Namespace: objMeta.Namespace,
-			},
+			GenerateName: artifact.Name + "-",
+			Namespace:    artifact.Namespace,
 		},
+		Spec: podSpec,
 	}
-
-	rb, err := r.clientSet.RbacV1().
-		RoleBindings(r.ArtifactPodInfo.Namespace).
-		Get(ctx, objMeta.Name, metav1.GetOptions{})
-	if rb == nil || apierrors.IsNotFound(err) {
-		_, err := r.clientSet.RbacV1().
-			RoleBindings(r.ArtifactPodInfo.Namespace).
-			Create(ctx, newrb, metav1.CreateOptions{})
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
 }
 
-// createRBAC creates a ServiceAccount, and a binding to the CopierRole so that
-// the container that copies the artifacts to the http server Pod has the
-// permissions to do so.
-func (r *OSArtifactReconciler) createRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	objMeta := genObjectMeta(artifact)
-
-	err := r.createCopierServiceAccount(ctx, objMeta)
-	if err != nil {
-		return errors.Wrap(err, "creating a service account")
-	}
-
-	err = r.createCopierRoleBinding(ctx, objMeta)
-	if err != nil {
-		return errors.Wrap(err, "creating a role binding for the copy-role")
-	}
-
-	return err
-}
-
-// removeRBAC deletes the role binding between the service account of this artifact
-// and the CopierRole. The ServiceAccount is removed automatically through the Owner
-// relationship with the OSArtifact. The RoleBinding can't have it as an owner
-// because it is in a different Namespace.
-func (r *OSArtifactReconciler) removeRBAC(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	err := r.clientSet.RbacV1().RoleBindings(r.ArtifactPodInfo.Namespace).
-		Delete(ctx, artifact.Name, metav1.DeleteOptions{})
-	// Ignore not found. No need to do anything.
-	if err != nil && apierrors.IsNotFound(err) {
-		return nil
-	}
-
-	return err
-}
-
-func (r *OSArtifactReconciler) removeArtifacts(ctx context.Context, artifact buildv1alpha1.OSArtifact) error {
-	//Finding Pods using labels
-	fmt.Printf("r.ArtifactPodInfo = %+v\n", r.ArtifactPodInfo.Label)
-	pods, err := r.clientSet.CoreV1().Pods(r.ArtifactPodInfo.Namespace).
-		List(ctx, metav1.ListOptions{LabelSelector: r.ArtifactPodInfo.Label})
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("listing pods with label %s in namespace %s", r.ArtifactPodInfo.Label, r.ArtifactPodInfo.Namespace))
-	}
-	if len(pods.Items) < 1 {
-		return errors.New("No artifact pod found")
-	}
-	pod := pods.Items[0]
-
-	stdout, stderr, err := r.executeRemoteCommand(r.ArtifactPodInfo.Namespace, pod.Name, fmt.Sprintf("rm -rf %s/%s.*", r.ArtifactPodInfo.Path, artifact.Name))
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("%s\n%s", stdout, stderr))
-	}
-	return nil
-}
-
-func (r *OSArtifactReconciler) executeRemoteCommand(namespace, podName, command string) (string, string, error) {
-	buf := &bytes.Buffer{}
-	errBuf := &bytes.Buffer{}
-	request := r.clientSet.CoreV1().RESTClient().
-		Post().
-		Namespace(namespace).
-		Resource("pods").
-		Name(podName).
-		SubResource("exec").
-		VersionedParams(&v1.PodExecOptions{
-			Command: []string{"/bin/sh", "-c", command},
-			Stdin:   false,
-			Stdout:  true,
-			Stderr:  true,
-			TTY:     true,
-		}, scheme.ParameterCodec)
-
-	exec, err := remotecommand.NewSPDYExecutor(r.restConfig, "POST", request.URL())
-	if err != nil {
-		return "", "", err
-	}
-	err = exec.Stream(remotecommand.StreamOptions{
-		Stdout: buf,
-		Stderr: errBuf,
-	})
-	if err != nil {
-		return "", "", fmt.Errorf("%w Failed executing command %s on %v/%v", err, command, namespace, podName)
-	}
-
-	return buf.String(), errBuf.String(), nil
+func ptr[T any](val T) *T {
+	return &val
 }

controllers/osartifact_controller.go

@@ -19,230 +19,286 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"time"
-
-	buildv1alpha1 "github.com/kairos-io/osbuilder/api/v1alpha1"
-	"github.com/pkg/errors"
+	osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"sigs.k8s.io/cluster-api/util/patch"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
-const FinalizerName = "build.kairos.io/osbuilder-finalizer"
-
-type ArtifactPodInfo struct {
-	Label     string
-	Namespace string
-	Path      string
-	Role      string
-}
+const (
+	FinalizerName                   = "build.kairos.io/osbuilder-finalizer"
+	artifactLabel                   = "build.kairos.io/artifact"
+	artifactExporterIndexAnnotation = "build.kairos.io/export-index"
+)
 
 // OSArtifactReconciler reconciles a OSArtifact object
 type OSArtifactReconciler struct {
 	client.Client
-	Scheme                               *runtime.Scheme
-	restConfig                           *rest.Config
-	clientSet                            *kubernetes.Clientset
 	ServingImage, ToolImage, CopierImage string
-	ArtifactPodInfo                      ArtifactPodInfo
 }
 
-func genObjectMeta(artifact buildv1alpha1.OSArtifact) metav1.ObjectMeta {
-	return metav1.ObjectMeta{
-		Name:            artifact.Name,
-		Namespace:       artifact.Namespace,
-		OwnerReferences: genOwner(artifact),
-	}
-}
+func (r *OSArtifactReconciler) InjectClient(c client.Client) error {
+	r.Client = c
 
-func genOwner(artifact buildv1alpha1.OSArtifact) []metav1.OwnerReference {
-	return []metav1.OwnerReference{
-		*metav1.NewControllerRef(&artifact.ObjectMeta, schema.GroupVersionKind{
-			Group:   buildv1alpha1.GroupVersion.Group,
-			Version: buildv1alpha1.GroupVersion.Version,
-			Kind:    "OSArtifact",
-		}),
-	}
+	return nil
 }
 
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=build.kairos.io,resources=osartifacts/finalizers,verbs=update
+//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;delete
+//+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;create;delete;watch
+//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;
+//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;create;
+//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete
 
-// TODO: Is this ^ how I should have created rbac permissions for the controller?
-//       - git commit all changes
-//       - generate code with kubebuilder
-//       - check if my permissions were removed
-//       - do it properly
-
-// Reconcile is part of the main kubernetes reconciliation loop which aims to
-// move the current state of the cluster closer to the desired state.
-// TODO(user): Modify the Reconcile function to compare the state specified by
-// the OSArtifact object against the actual cluster state, and then
-// perform operations to make the cluster state reflect the state specified by
-// the user.
-//
-// For more details, check Reconcile and its Result here:
-// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile
 func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 
-	var osbuild buildv1alpha1.OSArtifact
-	if err := r.Get(ctx, req.NamespacedName, &osbuild); err != nil {
+	var artifact osbuilder.OSArtifact
+	if err := r.Get(ctx, req.NamespacedName, &artifact); err != nil {
 		if apierrors.IsNotFound(err) {
 			return ctrl.Result{}, nil
 		}
-		return ctrl.Result{}, err
+		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Reconciling %v", osbuild))
-
-	stop, err := r.handleFinalizer(ctx, &osbuild)
-	if err != nil || stop {
-		return ctrl.Result{}, err
+	if artifact.DeletionTimestamp != nil {
+		controllerutil.RemoveFinalizer(&artifact, FinalizerName)
+		return ctrl.Result{}, r.Update(ctx, &artifact)
 	}
 
-	// generate configmap required for building a custom image
-	desiredConfigMap := r.genConfigMap(osbuild)
-	logger.Info(fmt.Sprintf("Checking configmap %v", osbuild))
-
-	cfgMap, err := r.clientSet.CoreV1().ConfigMaps(req.Namespace).Get(ctx, desiredConfigMap.Name, metav1.GetOptions{})
-	if cfgMap == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating config map %v", desiredConfigMap))
-
-		_, err = r.clientSet.CoreV1().ConfigMaps(req.Namespace).Create(ctx, desiredConfigMap, metav1.CreateOptions{})
-		if err != nil {
-			logger.Error(err, "Failed while creating config map")
-			return ctrl.Result{}, err
+	if !controllerutil.ContainsFinalizer(&artifact, FinalizerName) {
+		controllerutil.AddFinalizer(&artifact, FinalizerName)
+		if err := r.Update(ctx, &artifact); err != nil {
+			return ctrl.Result{Requeue: true}, err
 		}
+	}
+
+	logger.Info(fmt.Sprintf("Reconciling %s/%s", artifact.Namespace, artifact.Name))
+
+	switch artifact.Status.Phase {
+	case osbuilder.Exporting:
+		return r.checkExport(ctx, &artifact)
+	case osbuilder.Ready, osbuilder.Error:
+		return ctrl.Result{}, nil
+	default:
+		return r.checkBuild(ctx, &artifact)
+	}
+}
+
+func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	// generate configmap required for building a custom image
+	cm := r.genConfigMap(artifact)
+	if cm.Labels == nil {
+		cm.Labels = map[string]string{}
+	}
+	cm.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, cm, r.Scheme()); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
-	if err != nil {
+	if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Checking deployment %v", osbuild))
-
-	err = r.createRBAC(ctx, osbuild)
-	if err != nil {
+	pvc := r.newArtifactPVC(artifact)
+	if pvc.Labels == nil {
+		pvc.Labels = map[string]string{}
+	}
+	pvc.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, pvc, r.Scheme()); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+	if err := r.Create(ctx, pvc); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	desiredJob := r.genJob(osbuild)
-	job, err := r.clientSet.BatchV1().Jobs(req.Namespace).Get(ctx, desiredJob.Name, metav1.GetOptions{})
-	if job == nil || apierrors.IsNotFound(err) {
-		logger.Info(fmt.Sprintf("Creating Job %v", job))
+	pod := r.newBuilderPod(pvc.Name, artifact)
+	if pod.Labels == nil {
+		pod.Labels = map[string]string{}
+	}
+	pod.Labels[artifactLabel] = artifact.Name
+	if err := controllerutil.SetOwnerReference(artifact, pod, r.Scheme()); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+	if err := r.Create(ctx, pod); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
 
-		_, err = r.clientSet.BatchV1().Jobs(req.Namespace).Create(ctx, desiredJob, metav1.CreateOptions{})
-		if err != nil {
-			logger.Error(err, "Failed while creating job")
+	artifact.Status.Phase = osbuilder.Building
+	if err := r.Status().Update(ctx, artifact); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	var pods corev1.PodList
+	if err := r.List(ctx, &pods, &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labels.Set{
+			artifactLabel: artifact.Name,
+		}),
+	}); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	for _, pod := range pods.Items {
+		switch pod.Status.Phase {
+		case corev1.PodSucceeded:
+			artifact.Status.Phase = osbuilder.Exporting
+			return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
+		case corev1.PodFailed:
+			artifact.Status.Phase = osbuilder.Error
+			return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
+		case corev1.PodPending, corev1.PodRunning:
 			return ctrl.Result{}, nil
 		}
-
-		return ctrl.Result{Requeue: true}, nil
 	}
-	if err != nil {
+
+	return r.startBuild(ctx, artifact)
+}
+
+func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
+	var jobs batchv1.JobList
+	if err := r.List(ctx, &jobs, &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labels.Set{
+			artifactLabel: artifact.Name,
+		}),
+	}); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
 
-	logger.Info(fmt.Sprintf("Updating state %v", osbuild))
-
-	copy := osbuild.DeepCopy()
-
-	helper, err := patch.NewHelper(&osbuild, r.Client)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-	if job.Status.Succeeded > 0 {
-		copy.Status.Phase = "Ready"
-	} else if copy.Status.Phase != "Building" {
-		copy.Status.Phase = "Building"
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
-	defer cancel()
-	if err := helper.Patch(ctx, copy); err != nil {
-		return ctrl.Result{}, errors.Wrapf(err, "couldn't patch osbuild %q", copy.Name)
+	indexedJobs := make(map[string]*batchv1.Job, len(artifact.Spec.Exporters))
+	for _, job := range jobs.Items {
+		if job.GetAnnotations() != nil {
+			if idx, ok := job.GetAnnotations()[artifactExporterIndexAnnotation]; ok {
+				indexedJobs[idx] = &job
+			}
+		}
 	}
 
-	// for _, c := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
-	// 	if c.State.Terminated != nil && c.State.Terminated.ExitCode != 0 {
-	// 		packageBuildCopy.Status.State = "Failed"
-	// 	}
-	// }
+	var pvcs corev1.PersistentVolumeClaimList
+	var pvc *corev1.PersistentVolumeClaim
+	if err := r.List(ctx, &pvcs, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{artifactLabel: artifact.Name})}); err != nil {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	for _, item := range pvcs.Items {
+		pvc = &item
+		break
+	}
+
+	if pvc == nil {
+		log.FromContext(ctx).Error(nil, "failed to locate pvc for artifact, this should not happen")
+		return ctrl.Result{}, fmt.Errorf("failed to locate artifact pvc")
+	}
+
+	var succeeded int
+	for i := range artifact.Spec.Exporters {
+		idx := fmt.Sprintf("%d", i)
+
+		job := indexedJobs[idx]
+		if job == nil {
+			job = &batchv1.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      fmt.Sprintf("%s-export-%s", artifact.Name, idx),
+					Namespace: artifact.Namespace,
+					Annotations: map[string]string{
+						artifactExporterIndexAnnotation: idx,
+					},
+					Labels: map[string]string{
+						artifactLabel: artifact.Name,
+					},
+				},
+				Spec: artifact.Spec.Exporters[i],
+			}
+
+			job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
+				Name: "artifacts",
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: pvc.Name,
+						ReadOnly:  true,
+					},
+				},
+			})
+
+			if err := controllerutil.SetOwnerReference(artifact, job, r.Scheme()); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+
+			if err := r.Create(ctx, job); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+
+		} else if job.Spec.Completions == nil || *job.Spec.Completions == 1 {
+			if job.Status.Succeeded > 0 {
+				succeeded++
+			}
+		} else if *job.Spec.BackoffLimit <= job.Status.Failed {
+			artifact.Status.Phase = osbuilder.Error
+			if err := r.Status().Update(ctx, artifact); err != nil {
+				return ctrl.Result{Requeue: true}, err
+			}
+			break
+		}
+	}
+
+	if succeeded == len(artifact.Spec.Exporters) {
+		artifact.Status.Phase = osbuilder.Ready
+		if err := r.Status().Update(ctx, artifact); err != nil {
+			return ctrl.Result{Requeue: true}, err
+		}
+	}
 
 	return ctrl.Result{}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *OSArtifactReconciler) SetupWithManager(mgr ctrl.Manager) error {
-
-	cfg := mgr.GetConfig()
-	clientset, err := kubernetes.NewForConfig(cfg)
-	if err != nil {
-		return err
-	}
-	r.restConfig = cfg
-	r.clientSet = clientset
-
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&buildv1alpha1.OSArtifact{}).
+		For(&osbuilder.OSArtifact{}).
+		Owns(&osbuilder.OSArtifact{}).
+		Watches(
+			&source.Kind{Type: &corev1.Pod{}},
+			handler.EnqueueRequestsFromMapFunc(r.findOwningArtifact),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+		).
+		Watches(
+			&source.Kind{Type: &batchv1.Job{}},
+			handler.EnqueueRequestsFromMapFunc(r.findOwningArtifact),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+		).
 		Complete(r)
 }
 
-// Returns true if reconciliation should stop or false otherwise
-func (r *OSArtifactReconciler) handleFinalizer(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) (bool, error) {
-	// examine DeletionTimestamp to determine if object is under deletion
-	if osbuild.DeletionTimestamp.IsZero() {
-		// The object is not being deleted, so if it does not have our finalizer,
-		// then lets add the finalizer and update the object. This is equivalent
-		// registering our finalizer.
-		if !controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
-			controllerutil.AddFinalizer(osbuild, FinalizerName)
-			if err := r.Update(ctx, osbuild); err != nil {
-				return true, err
-			}
-		}
-	} else {
-		// The object is being deleted
-		if controllerutil.ContainsFinalizer(osbuild, FinalizerName) {
-			// our finalizer is present, so lets handle any external dependency
-			if err := r.finalize(ctx, osbuild); err != nil {
-				// if fail to delete the external dependency here, return with error
-				// so that it can be retried
-				return true, err
-			}
-
-			// remove our finalizer from the list and update it.
-			controllerutil.RemoveFinalizer(osbuild, FinalizerName)
-			if err := r.Update(ctx, osbuild); err != nil {
-				return true, err
-			}
-		}
-
-		// Stop reconciliation as the item is being deleted
-		return true, nil
+func (r *OSArtifactReconciler) findOwningArtifact(obj client.Object) []reconcile.Request {
+	if obj.GetLabels() == nil {
+		return nil
 	}
 
-	return false, nil
-}
-
-// - Remove artifacts from the server Pod
-// - Delete role-binding (because it doesn't have the OSArtifact as an owner and won't be deleted automatically)
-func (r *OSArtifactReconciler) finalize(ctx context.Context, osbuild *buildv1alpha1.OSArtifact) error {
-	if err := r.removeRBAC(ctx, *osbuild); err != nil {
-		return err
-	}
-
-	if err := r.removeArtifacts(ctx, *osbuild); err != nil {
-		return err
+	if artifactName, ok := obj.GetLabels()[artifactLabel]; ok {
+		return []reconcile.Request{
+			{
+				NamespacedName: types.NamespacedName{
+					Name:      artifactName,
+					Namespace: obj.GetNamespace(),
+				},
+			},
+		}
 	}
 
 	return nil