push image to registry

This commit is contained in:
Lukasz Zajaczkowski 2025-01-23 09:44:45 +01:00
parent b7de6843bb
commit ab540adbcd
11 changed files with 264 additions and 18034 deletions

View File

@ -17,8 +17,6 @@ limitations under the License.
package v1alpha2
import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -36,38 +34,18 @@ type OSArtifactSpec struct {
// Points to a prepared kairos image (e.g. a released one)
ImageName string `json:"imageName,omitempty"`
// Points to a vanilla (non-Kairos) image. osbuilder will try to convert this to a Kairos image
BaseImageName string `json:"baseImageName,omitempty"`
// Points to a Secret that contains a Dockerfile. osbuilder will build the image using that Dockerfile and will try to create a Kairos image from it.
BaseImageDockerfile *SecretKeySelector `json:"baseImageDockerfile,omitempty"`
ISO bool `json:"iso,omitempty"`
// +kubebuilder:validation:Type:=string
// +kubebuilder:validation:Enum:=rpi3;rpi4
Model *Model `json:"model,omitempty"`
//Disk-only stuff
DiskSize string `json:"diskSize,omitempty"`
CloudImage bool `json:"cloudImage,omitempty"`
AzureImage bool `json:"azureImage,omitempty"`
GCEImage bool `json:"gceImage,omitempty"`
Netboot bool `json:"netboot,omitempty"`
NetbootURL string `json:"netbootURL,omitempty"`
CloudConfigRef *SecretKeySelector `json:"cloudConfigRef,omitempty"`
GRUBConfig string `json:"grubConfig,omitempty"`
Bundles []string `json:"bundles,omitempty"`
FileBundles map[string]string `json:"fileBundles,omitempty"`
OSRelease string `json:"osRelease,omitempty"`
KairosRelease string `json:"kairosRelease,omitempty"`
Bundles []string `json:"bundles,omitempty"`
FileBundles map[string]string `json:"fileBundles,omitempty"`
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
Exporters []batchv1.JobSpec `json:"exporters,omitempty"`
Volume *corev1.PersistentVolumeClaimSpec `json:"volume,omitempty"`
OutputImage *OutputImage `json:"outputImage,omitempty"`
}
type SecretKeySelector struct {
@ -76,6 +54,14 @@ type SecretKeySelector struct {
Key string `json:"key,omitempty"`
}
type OutputImage struct {
Registry string `json:"registry,omitempty"`
Repository string `json:"repository,omitempty"`
Tag string `json:"tag,omitempty"`
Username string `json:"username,omitempty"`
PasswordSecretKeyRef *SecretKeySelector `json:"passwordSecretKeyRef,omitempty"`
}
type ArtifactPhase string
const (

View File

@ -21,9 +21,7 @@ limitations under the License.
package v1alpha2
import (
batchv1 "k8s.io/api/batch/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -89,11 +87,6 @@ func (in *OSArtifactList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) {
*out = *in
if in.BaseImageDockerfile != nil {
in, out := &in.BaseImageDockerfile, &out.BaseImageDockerfile
*out = new(SecretKeySelector)
**out = **in
}
if in.Model != nil {
in, out := &in.Model, &out.Model
*out = new(Model)
@ -116,21 +109,9 @@ func (in *OSArtifactSpec) DeepCopyInto(out *OSArtifactSpec) {
(*out)[key] = val
}
}
if in.ImagePullSecrets != nil {
in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
*out = make([]v1.LocalObjectReference, len(*in))
copy(*out, *in)
}
if in.Exporters != nil {
in, out := &in.Exporters, &out.Exporters
*out = make([]batchv1.JobSpec, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Volume != nil {
in, out := &in.Volume, &out.Volume
*out = new(v1.PersistentVolumeClaimSpec)
if in.OutputImage != nil {
in, out := &in.OutputImage, &out.OutputImage
*out = new(OutputImage)
(*in).DeepCopyInto(*out)
}
}
@ -150,7 +131,7 @@ func (in *OSArtifactStatus) DeepCopyInto(out *OSArtifactStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]metav1.Condition, len(*in))
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
@ -167,6 +148,26 @@ func (in *OSArtifactStatus) DeepCopy() *OSArtifactStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OutputImage) DeepCopyInto(out *OutputImage) {
*out = *in
if in.PasswordSecretKeyRef != nil {
in, out := &in.PasswordSecretKeyRef, &out.PasswordSecretKeyRef
*out = new(SecretKeySelector)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OutputImage.
func (in *OutputImage) DeepCopy() *OutputImage {
if in == nil {
return nil
}
out := new(OutputImage)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SecretKeySelector) DeepCopyInto(out *SecretKeySelector) {
*out = *in

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,62 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-registry
---
apiVersion: v1
kind: Service
metadata:
labels:
app: registry
name: osbuilder-registry
namespace: test-registry
spec:
ports:
- nodePort: 30100
port: 5000
protocol: TCP
targetPort: 5000
selector:
app: registry
type: NodePort
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: registry-pvc
namespace: test-registry
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 30Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: registry
name: registry
namespace: test-registry
spec:
replicas: 1
selector:
matchLabels:
app: registry
template:
metadata:
labels:
app: registry
spec:
containers:
- image: registry:2
imagePullPolicy: IfNotPresent
name: registry
volumeMounts:
- mountPath: /var/lib/registry
name: registry-vol
volumes:
- name: registry-vol
persistentVolumeClaim:
claimName: registry-pvc

View File

@ -1,7 +1,23 @@
kind: Secret
apiVersion: v1
metadata:
name: registry-config
namespace: osbuilder
stringData:
config.json: |
{
"auths": {
"osbuilder.plrl-dev-aws.onplural.sh": {
"auth": "CHANGE_ME"
}
}
}
---
kind: Secret
apiVersion: v1
metadata:
name: cloud-config
namespace: osbuilder
stringData:
userdata: |
#cloud-config
@ -39,13 +55,14 @@ stringData:
local_file: true
plural:
token: abc
token: CHANGE_ME
url: https://console.plrl-dev-aws.onplural.sh
---
kind: OSArtifact
apiVersion: build.kairos.io/v1alpha2
metadata:
name: kairos-plural
namespace: osbuilder
spec:
imageName: "quay.io/kairos/alpine:3.19-standard-arm64-rpi4-v3.2.4-k3sv1.31.3-k3s1"
iso: true
@ -57,22 +74,12 @@ spec:
cloudConfigRef:
name: cloud-config
key: userdata
exporters:
- template:
spec:
restartPolicy: Never
containers:
- name: upload
image: quay.io/curl/curl
command:
- /bin/sh
args:
- -c
- |
for f in $(ls /artifacts)
do
curl -T /artifacts/$f http://osartifactbuilder-operator-osbuilder-nginx/upload/$f
done
volumeMounts:
- name: artifacts
mountPath: /artifacts
outputImage:
registry: osbuilder.plrl-dev-aws.onplural.sh
repository: kairos
tag: latest
username: plural
passwordSecretKeyRef:
name: registry-config
key: config.json

View File

@ -1,36 +0,0 @@
/*
Copyright 2022.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (r *OSArtifactReconciler) genConfigMap(artifact *osbuilder.OSArtifact) *v1.ConfigMap {
return &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name,
Namespace: artifact.Namespace,
},
Data: map[string]string{
"grub.cfg": artifact.Spec.GRUBConfig,
"os-release": artifact.Spec.OSRelease,
}}
}

View File

@ -19,11 +19,10 @@ package controllers
import (
"fmt"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
)
func unpackContainer(id, containerImage, pullImage string) corev1.Container {
@ -50,105 +49,13 @@ func unpackContainer(id, containerImage, pullImage string) corev1.Container {
}
func unpackFileContainer(id, pullImage, name string) corev1.Container {
//var rootID int64 = 0
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: fmt.Sprintf("pull-image-%s", id),
Image: "gcr.io/go-containerregistry/crane:latest",
Command: []string{"crane"},
Args: []string{"--platform=linux/arm64", "pull", pullImage, fmt.Sprintf("/rootfs/oem/%s.tar", name)},
//SecurityContext: &corev1.SecurityContext{
// RunAsUser: &rootID,
// RunAsGroup: &rootID,
//},
Args: []string{"--platform=linux/arm64", "pull", pullImage, fmt.Sprintf("/rootfs/%s.tar", name)},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs/oem",
SubPath: "rootfs/oem",
},
},
}
}
func pushImageName(artifact *osbuilder.OSArtifact) string {
pushName := artifact.Spec.ImageName
if pushName != "" {
return pushName
}
return artifact.Name
}
func createImageContainer(containerImage string, artifact *osbuilder.OSArtifact) corev1.Container {
imageName := pushImageName(artifact)
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "create-image",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"tar -czvpf test.tar -C /rootfs . && luet util pack %[1]s test.tar %[2]s.tar && chmod +r %[2]s.tar && mv %[2]s.tar /artifacts",
imageName,
artifact.Name,
),
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
}
}
func osReleaseContainer(containerImage string) corev1.Container {
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "os-release",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
"cp -rfv /etc/os-release /rootfs/etc/os-release",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "config",
MountPath: "/etc/os-release",
SubPath: "os-release",
},
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
}
}
func kairosReleaseContainer(containerImage string) corev1.Container {
return corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "kairos-release",
Image: containerImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
"cp -rfv /etc/kairos-release /rootfs/etc/kairos-release",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "config",
MountPath: "/etc/kairos-release",
SubPath: "kairos-release",
},
{
Name: "artifacts",
MountPath: "/rootfs",
@ -159,8 +66,12 @@ func kairosReleaseContainer(containerImage string) corev1.Container {
}
func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *corev1.PersistentVolumeClaim {
if artifact.Spec.Volume == nil {
artifact.Spec.Volume = &corev1.PersistentVolumeClaimSpec{
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name + "-artifacts",
Namespace: artifact.Namespace,
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{
corev1.ReadWriteOnce,
},
@ -169,15 +80,7 @@ func (r *OSArtifactReconciler) newArtifactPVC(artifact *osbuilder.OSArtifact) *c
"storage": resource.MustParse(r.PVCStorage),
},
},
}
}
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: artifact.Name + "-artifacts",
Namespace: artifact.Namespace,
},
Spec: *artifact.Spec.Volume,
}
return pvc
@ -202,14 +105,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
},
}
if artifact.Spec.GRUBConfig != "" {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "config",
MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
SubPath: "grub.cfg",
})
}
cloudImgCmd := fmt.Sprintf(
"/raw-images.sh /rootfs /artifacts/%s.raw",
artifact.Name,
@ -225,16 +120,10 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
}
if artifact.Spec.CloudConfigRef != nil || artifact.Spec.GRUBConfig != "" {
cmd = fmt.Sprintf(
"auroraboot --debug build-iso --name %s --date=false --overlay-iso /iso/iso-overlay --output /artifacts dir:/rootfs",
artifact.Name,
)
}
if artifact.Spec.Model != nil {
cmd = fmt.Sprintf("/build-arm-image.sh --model %s --directory %s --recovery-partition-size 5120 --state-parition-size 6144 --size 16384 --images-size 4096 /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name)
if artifact.Spec.CloudConfigRef != nil {
cmd = fmt.Sprintf("/build-arm-image.sh --model %s --config /iso/iso-overlay/cloud_config.yaml --directory %s --recovery-partition-size 5120 --state-partition-size 6144 --size 16384 --images-size 4096 /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name)
cmd = fmt.Sprintf("/build-arm-image.sh --model %s --config /iso/iso-overlay/cloud_config.yaml --directory %s /artifacts/%s.iso", *artifact.Spec.Model, "/rootfs", artifact.Name)
}
}
@ -250,78 +139,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
VolumeMounts: volumeMounts,
}
buildCloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
cloudImgCmd,
},
VolumeMounts: volumeMounts,
}
if artifact.Spec.DiskSize != "" {
buildCloudImageContainer.Env = []corev1.EnvVar{{
Name: "EXTEND",
Value: artifact.Spec.DiskSize,
}}
}
extractNetboot := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-netboot",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Env: []corev1.EnvVar{{
Name: "URL",
Value: artifact.Spec.NetbootURL,
}},
Args: []string{
fmt.Sprintf(
"/netboot.sh /artifacts/%s.iso /artifacts/%s",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
buildAzureCloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-azure-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"/azure.sh /artifacts/%s.raw /artifacts/%s.vhd",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
buildGCECloudImageContainer := corev1.Container{
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{Privileged: ptr(true)},
Name: "build-gce-cloud-image",
Image: r.ToolImage,
Command: []string{"/bin/bash", "-cxe"},
Args: []string{
fmt.Sprintf(
"/gce.sh /artifacts/%s.raw /artifacts/%s.gce.raw",
artifact.Name,
artifact.Name,
),
},
VolumeMounts: volumeMounts,
}
podSpec := corev1.PodSpec{
AutomountServiceAccountToken: ptr(false),
RestartPolicy: corev1.RestartPolicyNever,
@ -348,17 +165,6 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
},
}
if artifact.Spec.BaseImageDockerfile != nil {
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
Name: "dockerfile",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: artifact.Spec.BaseImageDockerfile.Name,
},
},
})
}
if artifact.Spec.CloudConfigRef != nil {
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
Name: "cloudconfig",
@ -371,83 +177,13 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
})
}
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, artifact.Spec.ImagePullSecrets...)
podSpec.InitContainers = []corev1.Container{}
// Base image can be:
// - built from a dockerfile and converted to a kairos one
// - built by converting an existing image to a kairos one
// - a prebuilt kairos image
if artifact.Spec.BaseImageDockerfile != nil {
podSpec.InitContainers = append(podSpec.InitContainers, baseImageBuildContainers()...)
} else if artifact.Spec.BaseImageName != "" { // Existing base image - non kairos
podSpec.InitContainers = append(podSpec.InitContainers,
unpackContainer("baseimage-non-kairos", r.ToolImage, artifact.Spec.BaseImageName))
} else { // Existing Kairos base image
podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer("baseimage", r.ToolImage, artifact.Spec.ImageName))
}
// If base image was a non kairos one, either one we built with kaniko or prebuilt,
// convert it to a Kairos one, in a best effort manner.
if artifact.Spec.BaseImageDockerfile != nil || artifact.Spec.BaseImageName != "" {
podSpec.InitContainers = append(podSpec.InitContainers,
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "convert-to-kairos",
Image: "busybox",
Command: []string{"/bin/echo"},
Args: []string{"TODO"},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
})
}
for i, bundle := range artifact.Spec.Bundles {
podSpec.InitContainers = append(podSpec.InitContainers, unpackContainer(fmt.Sprint(i), r.ToolImage, bundle))
}
if artifact.Spec.OSRelease != "" {
podSpec.InitContainers = append(podSpec.InitContainers, osReleaseContainer(r.ToolImage))
}
if artifact.Spec.KairosRelease != "" {
podSpec.InitContainers = append(podSpec.InitContainers, kairosReleaseContainer(r.ToolImage))
}
if artifact.Spec.ISO || artifact.Spec.Netboot {
podSpec.Containers = append(podSpec.Containers, buildIsoContainer)
}
if artifact.Spec.Netboot {
podSpec.Containers = append(podSpec.Containers, extractNetboot)
}
if artifact.Spec.CloudImage {
podSpec.Containers = append(podSpec.Containers, buildCloudImageContainer)
}
if artifact.Spec.AzureImage {
podSpec.Containers = append(podSpec.Containers, buildAzureCloudImageContainer)
}
if artifact.Spec.GCEImage {
podSpec.Containers = append(podSpec.Containers, buildGCECloudImageContainer)
}
podSpec.Containers = append(podSpec.Containers, createImageContainer(r.ToolImage, artifact))
if artifact.Spec.ISO && artifact.Spec.Model != nil {
podSpec.InitContainers = []corev1.Container{}
podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{
Name: "create-directories",
Image: "busybox",
Command: []string{"sh", "-c", "mkdir -p /mnt/pv/artifacts && mkdir -p /mnt/pv/rootfs/oem && chown -R 65532:65532 /mnt/pv/artifacts && chown -R 65532:65532 /mnt/pv/rootfs && chown -R 65532:65532 /mnt/pv/rootfs/oem"},
Command: []string{"sh", "-c", "mkdir -p /mnt/pv/artifacts && chown -R 65532:65532 /mnt/pv/artifacts && mkdir -p /mnt/pv/rootfs && chown -R 65532:65532 /mnt/pv/rootfs"},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
@ -482,62 +218,3 @@ func (r *OSArtifactReconciler) newBuilderPod(pvcName string, artifact *osbuilder
func ptr[T any](val T) *T {
return &val
}
func baseImageBuildContainers() []corev1.Container {
return []corev1.Container{
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "kaniko-build",
Image: "gcr.io/kaniko-project/executor:latest",
Args: []string{
"--dockerfile", "dockerfile/Dockerfile",
"--context", "dir://workspace",
"--destination", "whatever", // We don't push, but it needs this
"--tar-path", "/rootfs/image.tar",
"--no-push",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
{
Name: "dockerfile",
MountPath: "/workspace/dockerfile",
},
},
},
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "image-extractor",
Image: "quay.io/luet/base",
Args: []string{
"util", "unpack", "--local", "file:////rootfs/image.tar", "/rootfs",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
},
corev1.Container{
ImagePullPolicy: corev1.PullAlways,
Name: "cleanup",
Image: "busybox",
Command: []string{"/bin/rm"},
Args: []string{
"/rootfs/image.tar",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/rootfs",
SubPath: "rootfs",
},
},
},
}
}

28
controllers/kubernetes.go Normal file
View File

@ -0,0 +1,28 @@
package controllers
import (
"context"
"fmt"
"reflect"
"k8s.io/client-go/util/retry"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
func TryToUpdateStatus(ctx context.Context, client ctrlruntimeclient.Client, object ctrlruntimeclient.Object) error {
key := ctrlruntimeclient.ObjectKeyFromObject(object)
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
original := object.DeepCopyObject().(ctrlruntimeclient.Object)
if err := client.Get(ctx, key, object); err != nil {
return fmt.Errorf("could not fetch current %s/%s state, got error: %+v", object.GetName(), object.GetNamespace(), err)
}
if reflect.DeepEqual(object, original) {
return nil
}
return client.Status().Patch(ctx, original, ctrlruntimeclient.MergeFrom(object))
})
}

View File

@ -80,7 +80,7 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{Requeue: true}, err
return ctrl.Result{}, err
}
if len(artifact.Status.Conditions) == 0 {
artifact.Status.Conditions = []metav1.Condition{}
@ -89,8 +89,8 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: ready,
Status: metav1.ConditionFalse,
})
if err := r.Status().Update(ctx, artifact); err != nil {
return ctrl.Result{Requeue: true}, err
if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
return ctrl.Result{}, err
}
}
@ -117,37 +117,19 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
Reason: ready,
Status: metav1.ConditionTrue,
})
return ctrl.Result{}, r.Status().Update(ctx, artifact)
return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case osbuilder.Error:
meta.SetStatusCondition(&artifact.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "Error",
})
return ctrl.Result{}, r.Status().Update(ctx, artifact)
return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
default:
return r.checkBuild(ctx, artifact)
}
}
// CreateConfigMap generates a configmap required for building a custom image
func (r *OSArtifactReconciler) CreateConfigMap(ctx context.Context, artifact *osbuilder.OSArtifact) error {
cm := r.genConfigMap(artifact)
if cm.Labels == nil {
cm.Labels = map[string]string{}
}
cm.Labels[artifactLabel] = artifact.Name
if err := controllerutil.SetOwnerReference(artifact, cm, r.Scheme); err != nil {
return err
}
if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) {
return err
}
return nil
}
func (r *OSArtifactReconciler) createPVC(ctx context.Context, artifact *osbuilder.OSArtifact) (*corev1.PersistentVolumeClaim, error) {
pvc := r.newArtifactPVC(artifact)
if pvc.Labels == nil {
@ -193,11 +175,6 @@ func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuild
}
}
err := r.CreateConfigMap(ctx, artifact)
if err != nil {
return ctrl.Result{Requeue: true}, err
}
pvc, err := r.createPVC(ctx, artifact)
if err != nil {
return ctrl.Result{Requeue: true}, err
@ -209,8 +186,8 @@ func (r *OSArtifactReconciler) startBuild(ctx context.Context, artifact *osbuild
}
artifact.Status.Phase = osbuilder.Building
if err := r.Status().Update(ctx, artifact); err != nil {
return ctrl.Result{Requeue: true}, err
if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
@ -223,17 +200,17 @@ func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuild
artifactLabel: artifact.Name,
}),
}); err != nil {
return ctrl.Result{Requeue: true}, err
return ctrl.Result{}, err
}
for _, pod := range pods.Items {
switch pod.Status.Phase {
case corev1.PodSucceeded:
artifact.Status.Phase = osbuilder.Exporting
return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case corev1.PodFailed:
artifact.Status.Phase = osbuilder.Error
return ctrl.Result{Requeue: true}, r.Status().Update(ctx, artifact)
return ctrl.Result{}, TryToUpdateStatus(ctx, r.Client, artifact)
case corev1.PodPending, corev1.PodRunning:
return ctrl.Result{}, nil
}
@ -243,6 +220,8 @@ func (r *OSArtifactReconciler) checkBuild(ctx context.Context, artifact *osbuild
}
func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuilder.OSArtifact) (ctrl.Result, error) {
logger := log.FromContext(ctx)
var jobs batchv1.JobList
if err := r.List(ctx, &jobs, &client.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set{
@ -253,7 +232,7 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
return ctrl.Result{Requeue: true}, nil
}
indexedJobs := make(map[string]*batchv1.Job, len(artifact.Spec.Exporters))
indexedJobs := make(map[string]*batchv1.Job, 1)
for _, job := range jobs.Items {
if job.GetAnnotations() != nil {
if idx, ok := job.GetAnnotations()[artifactExporterIndexAnnotation]; ok {
@ -279,9 +258,8 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
return ctrl.Result{}, fmt.Errorf("failed to locate artifact pvc")
}
var succeeded int
for i := range artifact.Spec.Exporters {
idx := fmt.Sprintf("%d", i)
if artifact.Spec.OutputImage != nil {
idx := fmt.Sprintf("%d", 1)
job := indexedJobs[idx]
if job == nil {
@ -296,15 +274,78 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
artifactLabel: artifact.Name,
},
},
Spec: artifact.Spec.Exporters[i],
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{
{
Name: "init-container",
Image: "busybox",
Command: []string{
"sh", "-c",
"echo -e 'FROM scratch\nWORKDIR /build\nCOPY *.iso /build' > /artifacts/Dockerfile",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
},
},
},
},
},
}
container := corev1.Container{
Name: "exporter",
Image: "gcr.io/kaniko-project/executor:latest",
Args: []string{
"--context=/artifacts/",
"--dockerfile=/artifacts/Dockerfile",
fmt.Sprintf("--destination=%s/%s:%s", artifact.Spec.OutputImage.Registry, artifact.Spec.OutputImage.Repository, artifact.Spec.OutputImage.Tag),
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "artifacts",
MountPath: "/artifacts",
SubPath: "artifacts",
},
},
}
if artifact.Spec.OutputImage != nil && artifact.Spec.OutputImage.PasswordSecretKeyRef != nil {
if err := r.Get(ctx, client.ObjectKey{Namespace: artifact.Namespace, Name: artifact.Spec.OutputImage.PasswordSecretKeyRef.Name}, &corev1.Secret{}); err != nil {
if errors.IsNotFound(err) {
logger.Info(fmt.Sprintf("Secret %s/%s not found", artifact.Namespace, artifact.Spec.OutputImage.PasswordSecretKeyRef.Name))
return requeue, nil
}
return ctrl.Result{}, err
}
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "docker-secret",
MountPath: "/kaniko/.docker",
})
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "docker-secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: artifact.Spec.OutputImage.PasswordSecretKeyRef.Name,
},
},
})
}
job.Spec.Template.Spec.Containers = append(job.Spec.Template.Spec.Containers, container)
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "artifacts",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.Name,
ReadOnly: true,
ReadOnly: false,
},
},
})
@ -321,23 +362,18 @@ func (r *OSArtifactReconciler) checkExport(ctx context.Context, artifact *osbuil
} else if job.Spec.Completions == nil || *job.Spec.Completions == 1 {
if job.Status.Succeeded > 0 {
succeeded++
artifact.Status.Phase = osbuilder.Ready
if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{}, err
}
}
} else if *job.Spec.BackoffLimit <= job.Status.Failed {
artifact.Status.Phase = osbuilder.Error
if err := r.Status().Update(ctx, artifact); err != nil {
if err := TryToUpdateStatus(ctx, r.Client, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{Requeue: true}, nil
return ctrl.Result{}, err
}
break
}
}
if succeeded == len(artifact.Spec.Exporters) {
artifact.Status.Phase = osbuilder.Ready
if err := r.Status().Update(ctx, artifact); err != nil {
log.FromContext(ctx).Error(err, "failed to update artifact status")
return ctrl.Result{Requeue: true}, nil
}
}

View File

@ -1,173 +0,0 @@
package controllers
import (
"context"
"fmt"
"time"
osbuilder "github.com/kairos-io/osbuilder/api/v1alpha2"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/phayes/freeport"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
)
var _ = Describe("OSArtifactReconciler", func() {
var r *OSArtifactReconciler
var artifact *osbuilder.OSArtifact
var namespace string
var restConfig *rest.Config
var clientset *kubernetes.Clientset
var err error
BeforeEach(func() {
restConfig = ctrl.GetConfigOrDie()
clientset, err = kubernetes.NewForConfig(restConfig)
Expect(err).ToNot(HaveOccurred())
namespace = createRandomNamespace(clientset)
artifact = &osbuilder.OSArtifact{
TypeMeta: metav1.TypeMeta{
Kind: "OSArtifact",
APIVersion: osbuilder.GroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: randStringRunes(10),
},
}
scheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(osbuilder.AddToScheme(scheme))
metricsPort, err := freeport.GetFreePort()
Expect(err).ToNot(HaveOccurred())
fmt.Printf("metricsPort = %+v\n", metricsPort)
mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
Scheme: scheme,
MetricsBindAddress: fmt.Sprintf("127.0.0.1:%d", metricsPort),
})
Expect(err).ToNot(HaveOccurred())
r = &OSArtifactReconciler{
ToolImage: "quay.io/kairos/auroraboot:latest",
}
err = (r).SetupWithManager(mgr)
Expect(err).ToNot(HaveOccurred())
})
JustBeforeEach(func() {
k8s := dynamic.NewForConfigOrDie(restConfig)
artifacts := k8s.Resource(
schema.GroupVersionResource{
Group: osbuilder.GroupVersion.Group,
Version: osbuilder.GroupVersion.Version,
Resource: "osartifacts"}).Namespace(namespace)
uArtifact := unstructured.Unstructured{}
uArtifact.Object, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(artifact)
resp, err := artifacts.Create(context.TODO(), &uArtifact, metav1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())
// Update the local object with the one fetched from k8s
err = runtime.DefaultUnstructuredConverter.FromUnstructured(resp.Object, artifact)
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
deleteNamepace(clientset, namespace)
})
Describe("CreateConfigMap", func() {
It("creates a ConfigMap with no error", func() {
ctx := context.Background()
err := r.CreateConfigMap(ctx, artifact)
Expect(err).ToNot(HaveOccurred())
c, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), artifact.Name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
Expect(c).ToNot(BeNil())
})
})
Describe("CreateBuilderPod", func() {
When("BaseImageDockerfile is set", func() {
BeforeEach(func() {
secretName := artifact.Name + "-dockerfile"
_, err := clientset.CoreV1().Secrets(namespace).Create(context.TODO(),
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
},
StringData: map[string]string{
"Dockerfile": "FROM ubuntu",
},
Type: "Opaque",
}, metav1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())
artifact.Spec.BaseImageDockerfile = &osbuilder.SecretKeySelector{
Name: secretName,
Key: "Dockerfile",
}
// Whatever, just to let it work
artifact.Spec.ImageName = "quay.io/kairos-ci/" + artifact.Name + ":latest"
})
It("creates an Init Container to build the image", func() {
pvc, err := r.createPVC(context.TODO(), artifact)
Expect(err).ToNot(HaveOccurred())
pod, err := r.createBuilderPod(context.TODO(), artifact, pvc)
Expect(err).ToNot(HaveOccurred())
By("checking if an init container was created")
initContainerNames := []string{}
for _, c := range pod.Spec.InitContainers {
initContainerNames = append(initContainerNames, c.Name)
}
Expect(initContainerNames).To(ContainElement("kaniko-build"))
By("checking if init containers complete successfully")
Eventually(func() bool {
p, err := clientset.CoreV1().Pods(namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
var allReady = false
if len(p.Status.InitContainerStatuses) > 0 {
allReady = true
}
for _, c := range p.Status.InitContainerStatuses {
allReady = allReady && c.Ready
}
return allReady
}, 2*time.Minute, 5*time.Second).Should(BeTrue())
// req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{})
// podLogs, err := req.Stream(context.TODO())
// Expect(err).ToNot(HaveOccurred())
// defer podLogs.Close()
// buf := new(bytes.Buffer)
// _, err = io.Copy(buf, podLogs)
// Expect(err).ToNot(HaveOccurred())
// str := buf.String()
// fmt.Printf("str = %+v\n", str)
})
})
})
})