push image to registry

This commit is contained in:
Lukasz Zajaczkowski 2025-01-23 09:44:45 +01:00
parent b7de6843bb
commit ab540adbcd
11 changed files with 264 additions and 18034 deletions

View File

@ -17,8 +17,6 @@ limitations under the License.
package v1alpha2 package v1alpha2
import ( import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
@ -36,38 +34,18 @@ type OSArtifactSpec struct {
// Points to a prepared kairos image (e.g. a released one) // Points to a prepared kairos image (e.g. a released one)
ImageName string `json:"imageName,omitempty"` ImageName string `json:"imageName,omitempty"`
// Points to a vanilla (non-Kairos) image. osbuilder will try to convert this to a Kairos image
BaseImageName string `json:"baseImageName,omitempty"`
// Points to a Secret that contains a Dockerfile. osbuilder will build the image using that Dockerfile and will try to create a Kairos image from it.
BaseImageDockerfile *SecretKeySelector `json:"baseImageDockerfile,omitempty"`
ISO bool `json:"iso,omitempty"` ISO bool `json:"iso,omitempty"`
// +kubebuilder:validation:Type:=string // +kubebuilder:validation:Type:=string
// +kubebuilder:validation:Enum:=rpi3;rpi4 // +kubebuilder:validation:Enum:=rpi3;rpi4
Model *Model `json:"model,omitempty"` Model *Model `json:"model,omitempty"`
//Disk-only stuff
DiskSize string `json:"diskSize,omitempty"`
CloudImage bool `json:"cloudImage,omitempty"`
AzureImage bool `json:"azureImage,omitempty"`
GCEImage bool `json:"gceImage,omitempty"`
Netboot bool `json:"netboot,omitempty"`
NetbootURL string `json:"netbootURL,omitempty"`
CloudConfigRef *SecretKeySelector `json:"cloudConfigRef,omitempty"` CloudConfigRef *SecretKeySelector `json:"cloudConfigRef,omitempty"`
GRUBConfig string `json:"grubConfig,omitempty"`
Bundles []string `json:"bundles,omitempty"` Bundles []string `json:"bundles,omitempty"`
FileBundles map[string]string `json:"fileBundles,omitempty"` FileBundles map[string]string `json:"fileBundles,omitempty"`
OSRelease string `json:"osRelease,omitempty"`
KairosRelease string `json:"kairosRelease,omitempty"`
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` OutputImage *OutputImage `json:"outputImage,omitempty"`
Exporters []batchv1.JobSpec `json:"exporters,omitempty"`
Volume *corev1.PersistentVolumeClaimSpec `json:"volume,omitempty"`
} }
type SecretKeySelector struct { type SecretKeySelector struct {
@ -76,6 +54,14 @@ type SecretKeySelector struct {
Key string `json:"key,omitempty"` Key string `json:"key,omitempty"`
} }
type OutputImage struct {
Registry string `json:"registry,omitempty"`
Repository string `json:"repository,omitempty"`
Tag string `json:"tag,omitempty"`
Username string `json:"username,omitempty"`
PasswordSecretKeyRef *SecretKeySelector `json:"passwordSecretKeyRef,omitempty"`
}
type ArtifactPhase string type ArtifactPhase string
const ( const (

View File

@ -21,9 +21,7 @@ limitations under the License.
package v1alpha2 package v1alpha2
import ( import (
batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime" runtime "k8s.io/apimachinery/pkg/runtime"
) )
@ -89,11 +87,6 @@ func (in *OSArtifactList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) { func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) {
*out = *in *out = *in
if in.BaseImageDockerfile != nil {
in, out := &in.BaseImageDockerfile, &out.BaseImageDockerfile
*out = new(SecretKeySelector)
**out = **in
}
if in.Model != nil { if in.Model != nil {
in, out := &in.Model, &out.Model in, out := &in.Model, &out.Model
*out = new(Model) *out = new(Model)
@ -116,21 +109,9 @@ func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) {
(*out)[key] = val (*out)[key] = val
} }
} }
if in.ImagePullSecrets != nil { if in.OutputImage != nil {
in, out := &in.ImagePullSecrets, &out.ImagePullSecrets in, out := &in.OutputImage, &out.OutputImage
*out = make([]v1.LocalObjectReference, len(*in)) *out = new(OutputImage)
copy(*out, *in)
}
if in.Exporters != nil {
in, out := &in.Exporters, &out.Exporters
*out = make([]batchv1.JobSpec, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Volume != nil {
in, out := &in.Volume, &out.Volume
*out = new(v1.PersistentVolumeClaimSpec)
(*in).DeepCopyInto(*out) (*in).DeepCopyInto(*out)
} }
} }
@ -150,7 +131,7 @@ func (in *OSArtifactStatus) DeepCopyInto(out *OSArtifactStatus) {
*out = *in *out = *in
if in.Conditions != nil { if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions in, out := &in.Conditions, &out.Conditions
*out = make([]metav1.Condition, len(*in)) *out = make([]v1.Condition, len(*in))
for i := range *in { for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i]) (*in)[i].DeepCopyInto(&(*out)[i])
} }
@ -167,6 +148,26 @@ func (in *OSArtifactStatus) DeepCopy() *OSArtifactStatus {
return out return out
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OutputImage) DeepCopyInto(out *OutputImage) {
*out = *in
if in.PasswordSecretKeyRef != nil {
in, out := &in.PasswordSecretKeyRef, &out.PasswordSecretKeyRef
*out = new(SecretKeySelector)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OutputImage.
func (in *OutputImage) DeepCopy() *OutputImage {
if in == nil {
return nil
}
out := new(OutputImage)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SecretKeySelector) DeepCopyInto(out *SecretKeySelector) { func (in *SecretKeySelector) DeepCopyInto(out *SecretKeySelector) {
*out = *in *out = *in

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,62 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-registry
---
apiVersion: v1
kind: Service
metadata:
labels:
app: registry
name: osbuilder-registry
namespace: test-registry
spec:
ports:
- nodePort: 30100
port: 5000
protocol: TCP
targetPort: 5000
selector:
app: registry
type: NodePort
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: registry-pvc
namespace: test-registry
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 30Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: registry
name: registry
namespace: test-registry
spec:
replicas: 1
selector:
matchLabels:
app: registry
template:
metadata:
labels:
app: registry
spec:
containers:
- image: registry:2
imagePullPolicy: IfNotPresent
name: registry
volumeMounts:
- mountPath: /var/lib/registry
name: registry-vol
volumes:
- name: registry-vol
persistentVolumeClaim:
claimName: registry-pvc

View File

@ -1,7 +1,23 @@
kind: Secret kind: Secret
apiVersion: v1 apiVersion: v1
metadata:
name: registry-config
namespace: osbuilder
stringData:
config.json: |
{
"auths": {
"osbuilder.plrl-dev-aws.onplural.sh": {
"auth": "CHANGE_ME"
}
}
}
---
kind: Secret
apiVersion: v1
metadata: metadata:
name: cloud-config name: cloud-config
namespace: osbuilder
stringData: stringData:
userdata: | userdata: |
#cloud-config #cloud-config
@ -39,13 +55,14 @@ stringData:
local_file: true local_file: true
plural: plural:
token: abc token: CHANGE_ME
url: https://console.plrl-dev-aws.onplural.sh url: https://console.plrl-dev-aws.onplural.sh
--- ---
kind: OSArtifact kind: OSArtifact
apiVersion: build.kairos.io/v1alpha2 apiVersion: build.kairos.io/v1alpha2
metadata: metadata:
name: kairos-plural name: kairos-plural
namespace: osbuilder
spec: spec:
imageName: "quay.io/kairos/alpine:3.19-standard-arm64-rpi4-v3.2.4-k3sv1.31.3-k3s1" imageName: "quay.io/kairos/alpine:3.19-standard-arm64-rpi4-v3.2.4-k3sv1.31.3-k3s1"
iso: true iso: true
@ -57,22 +74,12 @@ spec:
cloudConfigRef: cloudConfigRef:
name: cloud-config name: cloud-config
key: userdata key: userdata
exporters: outputImage:
- template: registry: osbuilder.plrl-dev-aws.onplural.sh
spec: repository: kairos
restartPolicy: Never tag: latest
containers: username: plural
- name: upload passwordSecretKeyRef:
image: quay.io/curl/curl name: registry-config
command: key: config.json
- /bin/sh
args:
- -c
- |
for f in $(ls /artifacts)
do
curl -T /artifacts/$f http://osartifactbuilder-operator-osbuilder-nginx/upload/$f
done
volumeMounts:
- name: artifacts
mountPath: /artifacts

View File

@ -1,36 +0,0 @@
/*
Copyright 2022.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (r *OSArtifactReconciler) genConfigMap(artifact *osbuilder.OSArtifact) *v1.ConfigMap {
return &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name,
Namespace: artifact.Namespace,
},
Data: map[string]string{
"grub.cfg": artifact.Spec.GRUBConfig,
"os-release": artifact.Spec.OSRelease,
}}
}

View File

@ -19,11 +19,10 @@ package controllers
import ( import (
"fmt" "fmt"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
) )
func unpackContainer(id, containerImage, pullImage string) corev1.Container { func unpackContainer(id, containerImage, pullImage string) corev1.Container {
@ -50,105 +49,13 @@ func unpackContainer(id, containerImage, pullImage string) corev1.Container {
} }
func unpackFileContainer(id, pullImage, name string) corev1.Container { func unpackFileContainer(id, pullImage, name string) corev1.Container {
//var rootID int64 = 0
return corev1.Container{ return corev1.Container{
ImagePullPolicy: corev1.PullAlways, ImagePullPolicy: corev1.PullAlways,
Name: fmt.Sprintf("pull-image-%s", id), Name: fmt.Sprintf("pull-image-%s", id),
Image: "gcr.io/go-containerregistry/crane:latest", Image: "gcr.io/go-containerregistry/crane:latest",
Command: []string{"crane"}, Command: []string{"crane"},
Args: []string{"--platform=linux/arm64", "pull", pullImage, fmt.Sprintf("/rootfs/oem/%s.tar", name)}, Args: []string{"--platform=linux/arm64", "pull", pullImage, fmt.Sprintf("/rootfs/%s.tar", name)},
//SecurityContext: &corev1.SecurityContext{
// RunAsUser: &rootID,
// RunAsGroup: &rootID,
//},
VolumeMounts: []corev1.VolumeMount{ VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs/oem",
SubPath: "rootfs/oem",
},
},
}
}
func pushImageName(artifact *osbuilder.OSArtifact) string {
pushName := artifact.Spec.ImageName
if pushName != "" {
return pushName
}
return artifact.Name
}
func createImageContainer(containerImage string, artifact *osbuilder.OSArtifact) corev1.Container {
imageName := pushImageName(artifact)
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "create-image",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"tar -czvpf test.tar -C /rootfs . && luet util pack %[1]s test.tar %[2]s.tar && chmod +r %[2]s.tar && mv %[2]s.tar /artifacts",
imageName,
artifact.Name,
),
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
}
}
func osReleaseContainer(containerImage string) corev1.Container {
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "os-release",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
"cp -rfv /etc/os-release /rootfs/etc/os-release",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "config",
MountPath: "/etc/os-release",
SubPath: "os-release",
},
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
}
}
func kairosReleaseContainer(containerImage string) corev1.Container {
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "kairos-release",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
"cp -rfv /etc/kairos-release /rootfs/etc/kairos-release",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "config",
MountPath: "/etc/kairos-release",
SubPath: "kairos-release",
},
{ {
Name: "artifacts", Name: "artifacts",
MountPath: "/rootfs", MountPath: "/rootfs",
@ -159,8 +66,12 @@ func kairosReleaseContainer(containerImage string) corev1.Container {
} }
func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *corev1.PersistentVolumeClaim { func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *corev1.PersistentVolumeClaim {
if artifact.Spec.Volume == nil { pvc := &corev1.PersistentVolumeClaim{
artifact.Spec.Volume = &corev1.PersistentVolumeClaimSpec{ ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name + "-artifacts",
Namespace: artifact.Namespace,
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{ AccessModes: []corev1.PersistentVolumeAccessMode{
corev1.ReadWriteOnce, corev1.ReadWriteOnce,
}, },
@ -169,15 +80,7 @@ func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *c
"storage": resource.MustParse(r.PVCStorage), "storage": resource.MustParse(r.PVCStorage),
}, },
}, },
}
}
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name + "-artifacts",
Namespace: artifact.Namespace,
}, },
Spec: *artifact.Spec.Volume,
} }
return pvc return pvc
@ -202,14 +105,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
}, },
} }
if artifact.Spec.GRUBConfig != "" {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "config",
MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
SubPath: "grub.cfg",
})
}
cloudImgCmd := fmt.Sprintf( cloudImgCmd := fmt.Sprintf(
"/raw-images.sh /rootfs /artifacts/%s.raw", "/raw-images.sh /rootfs /artifacts/%s.raw",
artifact.Name, artifact.Name,
@ -225,16 +120,10 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml" cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
} }
if artifact.Spec.CloudConfigRef != nil || artifact.Spec.GRUBConfig != "" {
cmd = fmt.Sprintf(
"auroraboot --debug build-iso --name %s --date=false --overlay-iso /iso/iso-overlay --output /artifacts dir:/rootfs",
artifact.Name,
)
}
if artifact.Spec.Model != nil { if artifact.Spec.Model != nil {
cmd = fmt.Sprintf("/build-arm-image.sh --model %s --directory %s --recovery-partition-size 5120 --state-parition-size 6144 --size 16384 --images-size 4096 /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name) cmd = fmt.Sprintf("/build-arm-image.sh --model %s --directory %s --recovery-partition-size 5120 --state-parition-size 6144 --size 16384 --images-size 4096 /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name)
if artifact.Spec.CloudConfigRef != nil { if artifact.Spec.CloudConfigRef != nil {
cmd = fmt.Sprintf("/build-arm-image.sh --model %s --config /iso/iso-overlay/cloud_config.yaml --directory %s --recovery-partition-size 5120 --state-partition-size 6144 --size 16384 --images-size 4096 /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name) cmd = fmt.Sprintf("/build-arm-image.sh --model %s --config /iso/iso-overlay/cloud_config.yaml --directory %s /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name)
} }
} }
@ -250,78 +139,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
VolumeMounts: volumeMounts, VolumeMounts: volumeMounts,
} }
buildCloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
cloudImgCmd,
},
VolumeMounts: volumeMounts,
}
if artifact.Spec.DiskSize != "" {
buildCloudImageContainer.Env = []corev1.EnvVar{{
Name: "EXTEND",
Value: artifact.Spec.DiskSize,
}}
}
extractNetboot := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-netboot",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Env: []corev1.EnvVar{{
Name: "URL",
Value: artifact.Spec.NetbootURL,
}},
Args: []string{
fmt.Sprintf(
"/netboot.sh /artifacts/%s.iso /artifacts/%s",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
buildAzureCloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-azure-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"/azure.sh /artifacts/%s.raw /artifacts/%s.vhd",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
buildGCECloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-gce-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"/gce.sh /artifacts/%s.raw /artifacts/%s.gce.raw",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
podSpec := corev1.PodSpec{ podSpec := corev1.PodSpec{
AutomountServiceAccountToken: ptr(false), AutomountServiceAccountToken: ptr(false),
RestartPolicy: corev1.RestartPolicyNever, RestartPolicy: corev1.RestartPolicyNever,
@ -348,17 +165,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
}, },
} }
if artifact.Spec.BaseImageDockerfile != nil {
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
Name: "dockerfile",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: artifact.Spec.BaseImageDockerfile.Name,
},
},
})
}
if artifact.Spec.CloudConfigRef != nil { if artifact.Spec.CloudConfigRef != nil {
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
Name: "cloudconfig", Name: "cloudconfig",
@ -371,83 +177,13 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
}) })
} }
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, artifact.Spec.ImagePullSecrets...)
podSpec.InitContainers = []corev1.Container{}
// Base image can be:
// - built from a dockerfile and converted to a kairos one
// - built by converting an existing image to a kairos one
// - a prebuilt kairos image
if artifact.Spec.BaseImageDockerfile != nil {
podSpec.InitContainers = append(podSpec.InitContainers, baseImageBuildContainers()...)
} else if artifact.Spec.BaseImageName != "" { // Existing base image - non kairos
podSpec.InitContainers = append(podSpec.InitContainers,
unpackContainer("baseimage-non-kairos", r.ToolImage, artifact.Spec.BaseImageName))
} else { // Existing Kairos base image
podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName))
}
// If base image was a non kairos one, either one we built with kaniko or prebuilt,
// convert it to a Kairos one, in a best effort manner.
if artifact.Spec.BaseImageDockerfile != nil || artifact.Spec.BaseImageName != "" {
podSpec.InitContainers = append(podSpec.InitContainers,
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "convert-to-kairos",
Image: "busybox",
Command: []string{"/bin/echo"},
Args: []string{"TODO"},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
})
}
for i, bundle := range artifact.Spec.Bundles {
podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle))
}
if artifact.Spec.OSRelease != "" {
podSpec.InitContainers = append(podSpec.InitContainers, osReleaseContainer(r.ToolImage))
}
if artifact.Spec.KairosRelease != "" {
podSpec.InitContainers = append(podSpec.InitContainers, kairosReleaseContainer(r.ToolImage))
}
if artifact.Spec.ISO || artifact.Spec.Netboot {
podSpec.Containers = append(podSpec.Containers, buildIsoContainer)
}
if artifact.Spec.Netboot {
podSpec.Containers = append(podSpec.Containers, extractNetboot)
}
if artifact.Spec.CloudImage {
podSpec.Containers = append(podSpec.Containers, buildCloudImageContainer)
}
if artifact.Spec.AzureImage {
podSpec.Containers = append(podSpec.Containers, buildAzureCloudImageContainer)
}
if artifact.Spec.GCEImage {
podSpec.Containers = append(podSpec.Containers, buildGCECloudImageContainer)
}
podSpec.Containers = append(podSpec.Containers, createImageContainer(r.ToolImage, artifact))
if artifact.Spec.ISO && artifact.Spec.Model != nil { if artifact.Spec.ISO && artifact.Spec.Model != nil {
podSpec.InitContainers = []corev1.Container{} podSpec.InitContainers = []corev1.Container{}
podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{
Name: "create-directories", Name: "create-directories",
Image: "busybox", Image: "busybox",
Command: []string{"sh", "-c", "mkdir -p /mnt/pv/artifacts && mkdir -p /mnt/pv/rootfs/oem && chown -R 65532:65532 /mnt/pv/artifacts && chown -R 65532:65532 /mnt/pv/rootfs && chown -R 65532:65532 /mnt/pv/rootfs/oem"}, Command: []string{"sh", "-c", "mkdir -p /mnt/pv/artifacts && chown -R 65532:65532 /mnt/pv/artifacts && mkdir -p /mnt/pv/rootfs && chown -R 65532:65532 /mnt/pv/rootfs"},
VolumeMounts: []corev1.VolumeMount{ VolumeMounts: []corev1.VolumeMount{
{ {
Name: "artifacts", Name: "artifacts",
@ -482,62 +218,3 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
func ptr[T any](val T) *T { func ptr[T any](val T) *T {
return &val return &val
} }
func baseImageBuildContainers() []corev1.Container {
return []corev1.Container{
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "kaniko-build",
Image: "gcr.io/kaniko-project/executor:latest",
Args: []string{
"--dockerfile", "dockerfile/Dockerfile",
"--context", "dir://workspace",
"--destination", "whatever", // We don't push, but it needs this
"--tar-path", "/rootfs/image.tar",
"--no-push",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
{
Name: "dockerfile",
MountPath: "/workspace/dockerfile",
},
},
},
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "image-extractor",
Image: "quay.io/luet/base",
Args: []string{
"util", "unpack", "--local", "file:////rootfs/image.tar", "/rootfs",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
},
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "cleanup",
Image: "busybox",
Command: []string{"/bin/rm"},
Args: []string{
"/rootfs/image.tar",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
},
}
}

28
controllers/kubernetes.go Normal file
View File

@ -0,0 +1,28 @@
package controllers
import (
"context"
"fmt"
"reflect"
"k8s.io/client-go/util/retry"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
func TryToUpdateStatus(ctx context.Context, client ctrlruntimeclient.Client, object ctrlruntimeclient.Object) error {
key := ctrlruntimeclient.ObjectKeyFromObject(object)
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
original := object.DeepCopyObject().(ctrlruntimeclient.Object)
if err := client.Get(ctx, key, object); err != nil {
return fmt.Errorf("could not fetch current %s/%s state, got error: %+v", object.GetName(), object.GetNamespace(), err)
}
if reflect.DeepEqual(object, original) {
return nil
}
return client.Status().Patch(ctx, original, ctrlruntimeclient.MergeFrom(object))
})
}

View File

@ -80,7 +80,7 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if apierrors.IsNotFound(err) { if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil return ctrl.Result{}, nil
} }
return ctrl.Result{Requeue: true}, err return ctrl.Result{}, err
} }
if len(artifact.Status.Conditions) == 0 { if len(artifact.Status.Conditions) == 0 {
artifact.Status.Conditions = []metav1.Condition{} artifact.Status.Conditions = []metav1.Condition{}
@ -89,8 +89,8 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: ready, Reason: ready,
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
}) })
if err := r.Status().Update(ctx, artifact); err != nil { if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
return ctrl.Result{Requeue: true}, err return ctrl.Result{}, err
} }
} }
@ -117,37 +117,19 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: ready, Reason: ready,
Status: metav1.ConditionTrue, Status: metav1.ConditionTrue,
}) })
return ctrl.Result{}, r.Status().Update(ctx, artifact) return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case osbuilder.Error: case osbuilder.Error:
meta.SetStatusCondition(&artifact.Status.Conditions, metav1.Condition{ meta.SetStatusCondition(&artifact.Status.Conditions, metav1.Condition{
Type: "Ready", Type: "Ready",
Status: metav1.ConditionFalse, Status: metav1.ConditionFalse,
Reason: "Error", Reason: "Error",
}) })
return ctrl.Result{}, r.Status().Update(ctx, artifact) return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
default: default:
return r.checkBuild(ctx, artifact) return r.checkBuild(ctx, artifact)
} }
} }
// CreateConfigMap generates a configmap required for building a custom image
func (r *OSArtifactReconciler) CreateConfigMap(ctx context.Context, artifact *osbuilder.OSArtifact) error {
cm := r.genConfigMap(artifact)
if cm.Labels == nil {
cm.Labels = map[string]string{}
}
cm.Labels[artifactLabel] = artifact.Name
if err := controllerutil.SetOwnerReference(artifact, cm, r.Scheme); err != nil {
return err
}
if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (r *OSArtifactReconciler) createPVC(ctx context.Context, artifact *osbuilder.OSArtifact) (*corev1.PersistentVolumeClaim, error) { func (r *OSArtifactReconciler) createPVC(ctx context.Context, artifact *osbuilder.OSArtifact) (*corev1.PersistentVolumeClaim, error) {
pvc := r.newArtifactPVC(artifact) pvc := r.newArtifactPVC(artifact)
if pvc.Labels == nil { if pvc.Labels == nil {
@ -193,11 +175,6 @@ func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuild
} }
} }
err := r.CreateConfigMap(ctx, artifact)
if err != nil {
return ctrl.Result{Requeue: true}, err
}
pvc, err := r.createPVC(ctx, artifact) pvc, err := r.createPVC(ctx, artifact)
if err != nil { if err != nil {
return ctrl.Result{Requeue: true}, err return ctrl.Result{Requeue: true}, err
@ -209,8 +186,8 @@ func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuild
} }
artifact.Status.Phase = osbuilder.Building artifact.Status.Phase = osbuilder.Building
if err := r.Status().Update(ctx, artifact); err != nil { if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
return ctrl.Result{Requeue: true}, err return ctrl.Result{}, err
} }
return ctrl.Result{}, nil return ctrl.Result{}, nil
@ -223,17 +200,17 @@ func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuild
artifactLabel: artifact.Name, artifactLabel: artifact.Name,
}), }),
}); err != nil { }); err != nil {
return ctrl.Result{Requeue: true}, err return ctrl.Result{}, err
} }
for _, pod := range pods.Items { for _, pod := range pods.Items {
switch pod.Status.Phase { switch pod.Status.Phase {
case corev1.PodSucceeded: case corev1.PodSucceeded:
artifact.Status.Phase = osbuilder.Exporting artifact.Status.Phase = osbuilder.Exporting
return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact) return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case corev1.PodFailed: case corev1.PodFailed:
artifact.Status.Phase = osbuilder.Error artifact.Status.Phase = osbuilder.Error
return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact) return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case corev1.PodPending, corev1.PodRunning: case corev1.PodPending, corev1.PodRunning:
return ctrl.Result{}, nil return ctrl.Result{}, nil
} }
@ -243,6 +220,8 @@ func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuild
} }
func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) { func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
logger := log.FromContext(ctx)
var jobs batchv1.JobList var jobs batchv1.JobList
if err := r.List(ctx, &jobs, &client.ListOptions{ if err := r.List(ctx, &jobs, &client.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set{ LabelSelector: labels.SelectorFromSet(labels.Set{
@ -253,7 +232,7 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
return ctrl.Result{Requeue: true}, nil return ctrl.Result{Requeue: true}, nil
} }
indexedJobs := make(map[string]*batchv1.Job, len(artifact.Spec.Exporters)) indexedJobs := make(map[string]*batchv1.Job, 1)
for _, job := range jobs.Items { for _, job := range jobs.Items {
if job.GetAnnotations() != nil { if job.GetAnnotations() != nil {
if idx, ok := job.GetAnnotations()[artifactExporterIndexAnnotation]; ok { if idx, ok := job.GetAnnotations()[artifactExporterIndexAnnotation]; ok {
@ -279,9 +258,8 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
return ctrl.Result{}, fmt.Errorf("failed to locate artifact pvc") return ctrl.Result{}, fmt.Errorf("failed to locate artifact pvc")
} }
var succeeded int if artifact.Spec.OutputImage != nil {
for i := range artifact.Spec.Exporters { idx := fmt.Sprintf("%d", 1)
idx := fmt.Sprintf("%d", i)
job := indexedJobs[idx] job := indexedJobs[idx]
if job == nil { if job == nil {
@ -296,15 +274,78 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
artifactLabel: artifact.Name, artifactLabel: artifact.Name,
}, },
}, },
Spec: artifact.Spec.Exporters[i], Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{
{
Name: "init-container",
Image: "busybox",
Command: []string{
"sh", "-c",
"echo -e 'FROM scratch\nWORKDIR /build\nCOPY *.iso /build' > /artifacts/Dockerfile",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
},
},
},
},
},
} }
container := corev1.Container{
Name: "exporter",
Image: "gcr.io/kaniko-project/executor:latest",
Args: []string{
"--context=/artifacts/",
"--dockerfile=/artifacts/Dockerfile",
fmt.Sprintf("--destination=%s/%s:%s", artifact.Spec.OutputImage.Registry, artifact.Spec.OutputImage.Repository, artifact.Spec.OutputImage.Tag),
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
}
if artifact.Spec.OutputImage != nil && artifact.Spec.OutputImage.PasswordSecretKeyRef != nil {
if err := r.Get(ctx, client.ObjectKey{Namespace: artifact.Namespace, Name: artifact.Spec.OutputImage.PasswordSecretKeyRef.Name}, &corev1.Secret{}); err != nil {
if errors.IsNotFound(err) {
logger.Info(fmt.Sprintf("Secret %s/%s not found", artifact.Namespace, artifact.Spec.OutputImage.PasswordSecretKeyRef.Name))
return requeue, nil
}
return ctrl.Result{}, err
}
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "docker-secret",
MountPath: "/kaniko/.docker",
})
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "docker-secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: artifact.Spec.OutputImage.PasswordSecretKeyRef.Name,
},
},
})
}
job.Spec.Template.Spec.Containers = append(job.Spec.Template.Spec.Containers, container)
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "artifacts", Name: "artifacts",
VolumeSource: corev1.VolumeSource{ VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.Name, ClaimName: pvc.Name,
ReadOnly: true, ReadOnly: false,
}, },
}, },
}) })
@ -321,23 +362,18 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
} else if job.Spec.Completions == nil || *job.Spec.Completions == 1 { } else if job.Spec.Completions == nil || *job.Spec.Completions == 1 {
if job.Status.Succeeded > 0 { if job.Status.Succeeded > 0 {
succeeded++ artifact.Status.Phase = osbuilder.Ready
if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{}, err
}
} }
} else if *job.Spec.BackoffLimit <= job.Status.Failed { } else if *job.Spec.BackoffLimit <= job.Status.Failed {
artifact.Status.Phase = osbuilder.Error artifact.Status.Phase = osbuilder.Error
if err := r.Status().Update(ctx, artifact); err != nil { if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status") log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{Requeue: true}, nil return ctrl.Result{}, err
} }
break
}
}
if succeeded == len(artifact.Spec.Exporters) {
artifact.Status.Phase = osbuilder.Ready
if err := r.Status().Update(ctx, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{Requeue: true}, nil
} }
} }

View File

@ -1,173 +0,0 @@
package controllers
import (
"context"
"fmt"
"time"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/phayes/freeport"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
)
var _ = Describe("OSArtifactReconciler", func() {
var r *OSArtifactReconciler
var artifact *osbuilder.OSArtifact
var namespace string
var restConfig *rest.Config
var clientset *kubernetes.Clientset
var err error
BeforeEach(func() {
restConfig = ctrl.GetConfigOrDie()
clientset, err = kubernetes.NewForConfig(restConfig)
Expect(err).ToNot(HaveOccurred())
namespace = createRandomNamespace(clientset)
artifact = &osbuilder.OSArtifact{
TypeMeta: metav1.TypeMeta{
Kind: "OSArtifact",
APIVersion: osbuilder.GroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: randStringRunes(10),
},
}
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(osbuilder.AddToScheme(scheme))
metricsPort, err := freeport.GetFreePort()
Expect(err).ToNot(HaveOccurred())
fmt.Printf("metricsPort = %+v\n", metricsPort)
mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
Scheme: scheme,
MetricsBindAddress: fmt.Sprintf("127.0.0.1:%d", metricsPort),
})
Expect(err).ToNot(HaveOccurred())
r = &OSArtifactReconciler{
ToolImage: "quay.io/kairos/auroraboot:latest",
}
err = (r).SetupWithManager(mgr)
Expect(err).ToNot(HaveOccurred())
})
JustBeforeEach(func() {
k8s := dynamic.NewForConfigOrDie(restConfig)
artifacts := k8s.Resource(
schema.GroupVersionResource{
Group: osbuilder.GroupVersion.Group,
Version: osbuilder.GroupVersion.Version,
Resource: "osartifacts"}).Namespace(namespace)
uArtifact := unstructured.Unstructured{}
uArtifact.Object, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(artifact)
resp, err := artifacts.Create(context.TODO(), &uArtifact, metav1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())
// Update the local object with the one fetched from k8s
err = runtime.DefaultUnstructuredConverter.FromUnstructured(resp.Object, artifact)
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
deleteNamepace(clientset, namespace)
})
Describe("CreateConfigMap", func() {
It("creates a ConfigMap with no error", func() {
ctx := context.Background()
err := r.CreateConfigMap(ctx, artifact)
Expect(err).ToNot(HaveOccurred())
c, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), artifact.Name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
Expect(c).ToNot(BeNil())
})
})
Describe("CreateBuilderPod", func() {
When("BaseImageDockerfile is set", func() {
BeforeEach(func() {
secretName := artifact.Name + "-dockerfile"
_, err := clientset.CoreV1().Secrets(namespace).Create(context.TODO(),
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
},
StringData: map[string]string{
"Dockerfile": "FROM ubuntu",
},
Type: "Opaque",
}, metav1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())
artifact.Spec.BaseImageDockerfile = &osbuilder.SecretKeySelector{
Name: secretName,
Key: "Dockerfile",
}
// Whatever, just to let it work
artifact.Spec.ImageName = "quay.io/kairos-ci/" + artifact.Name + ":latest"
})
It("creates an Init Container to build the image", func() {
pvc, err := r.createPVC(context.TODO(), artifact)
Expect(err).ToNot(HaveOccurred())
pod, err := r.createBuilderPod(context.TODO(), artifact, pvc)
Expect(err).ToNot(HaveOccurred())
By("checking if an init container was created")
initContainerNames := []string{}
for _, c := range pod.Spec.InitContainers {
initContainerNames = append(initContainerNames, c.Name)
}
Expect(initContainerNames).To(ContainElement("kaniko-build"))
By("checking if init containers complete successfully")
Eventually(func() bool {
p, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
var allReady = false
if len(p.Status.InitContainerStatuses) > 0 {
allReady = true
}
for _, c := range p.Status.InitContainerStatuses {
allReady = allReady && c.Ready
}
return allReady
}, 2*time.Minute, 5*time.Second).Should(BeTrue())
// req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{})
// podLogs, err := req.Stream(context.TODO())
// Expect(err).ToNot(HaveOccurred())
// defer podLogs.Close()
// buf := new(bytes.Buffer)
// _, err = io.Copy(buf, podLogs)
// Expect(err).ToNot(HaveOccurred())
// str := buf.String()
// fmt.Printf("str = %+v\n", str)
})
})
})
})