Add scripts to generate raw images

See: https://github.com/kairos-io/kairos/issues/377

Makefile

@@ -276,10 +276,4 @@ unit-tests: test_deps
 e2e-tests:
 	GINKGO=$(GINKGO) KUBE_VERSION=${KUBE_VERSION} $(ROOT_DIR)/script/test.sh
 
-kind-e2e-tests: ginkgo kind-setup install undeploy-dev deploy-dev e2e-tests
-
-
-kubesplit: manifests kustomize
-	rm -rf helm-chart
-	mkdir helm-chart
-	$(KUSTOMIZE) build config/default | kubesplit -helm helm-chart
+kind-e2e-tests: ginkgo kind-setup install undeploy-dev deploy-dev e2e-tests

api/v1alpha1/osartifact_types.go

@@ -30,18 +30,7 @@ type OSArtifactSpec struct {
 
 	// Foo is an example field of OSArtifact. Edit osartifact_types.go to remove/update
 	ImageName string `json:"imageName,omitempty"`
-	// This needs to be revisited
-	ISO bool `json:"iso,omitempty"`
-
-	//Disk-only stuff
-	DiskSize   string `json:"diskSize,omitempty"`
-	CloudImage bool   `json:"cloudImage,omitempty"`
-	AzureImage bool   `json:"azureImage,omitempty"`
-	GCEImage   bool   `json:"gceImage,omitempty"`
-
-	Netboot    bool   `json:"netboot,omitempty"`
-	NetbootURL string `json:"netbootURL,omitempty"`
-
+	ISO       bool   `json:"iso,omitempty"`
 	// TODO: treat cloudconfig as a secret, and take a secretRef where to store it (optionally)
 	CloudConfig string `json:"cloudConfig,omitempty"`
 	GRUBConfig  string `json:"grubConfig,omitempty"`

config/crd/bases/build.c3os-x.io_osartifacts.yaml

@@ -35,8 +35,6 @@ spec:
           spec:
             description: OSArtifactSpec defines the desired state of OSArtifact
             properties:
-              azureImage:
-                type: boolean
               bundles:
                 items:
                   type: string
@@ -45,13 +43,6 @@ spec:
                 description: 'TODO: treat cloudconfig as a secret, and take a secretRef
                   where to store it (optionally)'
                 type: string
-              cloudImage:
-                type: boolean
-              diskSize:
-                description: Disk-only stuff
-                type: string
-              gceImage:
-                type: boolean
               grubConfig:
                 type: string
               imageName:
@@ -59,12 +50,7 @@ spec:
                   to remove/update
                 type: string
               iso:
-                description: This needs to be revisited
                 type: boolean
-              netboot:
-                type: boolean
-              netbootURL:
-                type: string
               osRelease:
                 type: string
               pull:

controllers/deployment.go

@@ -104,8 +104,7 @@ func osReleaseContainer(containerImage string) v1.Container {
 	}
 }
 
-func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact, svc *v1.Service) *appsv1.Deployment {
-	// TODO: svc is unused, but could be used in the future to generate the Netboot URL
+func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact) *appsv1.Deployment {
 	objMeta := metav1.ObjectMeta{
 		Name:            artifact.Name,
 		Namespace:       artifact.Namespace,
@@ -141,19 +140,12 @@ func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact,
 		})
 	}
 
-	cloudImgCmd := fmt.Sprintf(
-		"/raw-images.sh /rootfs /public/%s.raw",
-		artifact.Name,
-	)
-
 	if artifact.Spec.CloudConfig != "" {
 		volumeMounts = append(volumeMounts, v1.VolumeMount{
 			Name:      "config",
 			MountPath: "/iso/iso-overlay/cloud_config.yaml",
 			SubPath:   "config",
 		})
-
-		cloudImgCmd += " /iso/iso-overlay/cloud_config.yaml"
 	}
 
 	if artifact.Spec.CloudConfig != "" || artifact.Spec.GRUBConfig != "" {
@@ -175,78 +167,6 @@ func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact,
 		VolumeMounts: volumeMounts,
 	}
 
-	buildCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "build-cloud-image",
-		Image:           r.ToolImage,
-
-		Command: []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			cloudImgCmd,
-		},
-		VolumeMounts: volumeMounts,
-	}
-
-	if artifact.Spec.DiskSize != "" {
-		buildCloudImageContainer.Env = []v1.EnvVar{{
-			Name:  "EXTEND",
-			Value: artifact.Spec.DiskSize,
-		}}
-	}
-
-	extractNetboot := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "build-netboot",
-		Image:           r.ToolImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Env: []v1.EnvVar{{
-			Name:  "URL",
-			Value: artifact.Spec.NetbootURL,
-		}},
-		Args: []string{
-			fmt.Sprintf(
-				"/netboot.sh /public/%s.iso /public/%s",
-				artifact.Name,
-				artifact.Name,
-			),
-		},
-		VolumeMounts: volumeMounts,
-	}
-
-	buildAzureCloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "build-azure-cloud-image",
-		Image:           r.ToolImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			fmt.Sprintf(
-				"/azure.sh /public/%s.raw /public/%s.vhd",
-				artifact.Name,
-				artifact.Name,
-			),
-		},
-		VolumeMounts: volumeMounts,
-	}
-
-	buildGCECloudImageContainer := v1.Container{
-		ImagePullPolicy: v1.PullAlways,
-		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
-		Name:            "build-gce-cloud-image",
-		Image:           r.ToolImage,
-		Command:         []string{"/bin/bash", "-cxe"},
-		Args: []string{
-			fmt.Sprintf(
-				"/gce.sh /public/%s.raw /public/%s.gce.raw",
-				artifact.Name,
-				artifact.Name,
-			),
-		},
-		VolumeMounts: volumeMounts,
-	}
-
 	servingContainer := v1.Container{
 		ImagePullPolicy: v1.PullAlways,
 		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
@@ -292,28 +212,11 @@ func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact,
 
 	}
 
-	if artifact.Spec.ISO || artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
-	}
-
-	if artifact.Spec.Netboot {
-		pod.InitContainers = append(pod.InitContainers, extractNetboot)
-	}
-
-	if artifact.Spec.CloudImage || artifact.Spec.AzureImage || artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildCloudImageContainer)
-	}
-
-	if artifact.Spec.AzureImage {
-		pod.InitContainers = append(pod.InitContainers, buildAzureCloudImageContainer)
-	}
-
-	if artifact.Spec.GCEImage {
-		pod.InitContainers = append(pod.InitContainers, buildGCECloudImageContainer)
-	}
+	pod.InitContainers = append(pod.InitContainers, buildIsoContainer)
 
 	if pushImage {
 		pod.InitContainers = append(pod.InitContainers, createImageContainer(r.ToolImage, artifact.Spec.PushOptions))
+
 	}
 
 	pod.Containers = []v1.Container{servingContainer}

controllers/osartifact_controller.go

@@ -118,7 +118,7 @@ func (r *OSArtifactReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 	logger.Info(fmt.Sprintf("Checking deployment %v", osbuild))
 
-	desiredDeployment := r.genDeployment(osbuild, svc)
+	desiredDeployment := r.genDeployment(osbuild)
 	deployment, err := r.clientSet.AppsV1().Deployments(req.Namespace).Get(ctx, desiredDeployment.Name, v1.GetOptions{})
 	if deployment == nil || apierrors.IsNotFound(err) {
 		logger.Info(fmt.Sprintf("Creating Deployment %v", deployment))

go.sum

@@ -189,7 +189,6 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I=
 github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8=
@@ -278,7 +277,6 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -828,7 +826,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y=
 golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

tools-image/Dockerfile

@@ -1,4 +1,4 @@
-ARG ELEMENTAL_CLI_VERSION=0.20221107.1
+ARG ELEMENTAL_CLI_VERSION=0.20221020.3
 ARG LEAP_VERSION=15.4
 ARG LUET_VERSION=0.33.0
 FROM quay.io/kairos/packages:elemental-cli-system-$ELEMENTAL_CLI_VERSION AS elemental
@@ -70,8 +70,6 @@ RUN mkdir /config
 RUN zypper in -y jq docker git curl gptfdisk kpartx sudo
 # Netboot
 RUN zypper in -y cdrtools
-# cloud images
-RUN zypper in -y bc qemu-tools
 
 # ISO build config
 COPY ./config.yaml /config/manifest.yaml
@@ -86,7 +84,6 @@ COPY ./arm /arm
 COPY ./gce.sh /gce.sh
 COPY ./raw-images.sh /raw-images.sh
 COPY ./azure.sh /azure.sh
-COPY ./netboot.sh /netboot.sh
 
 COPY defaults.yaml /defaults.yaml
 

tools-image/azure.sh

@@ -3,15 +3,13 @@
 # Transform a raw image disk to azure vhd
 RAWIMAGE="$1"
 VHDDISK="${2:-disk.vhd}"
-cp -rf $RAWIMAGE $VHDDISK.work
 
 MB=$((1024*1024))
 size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
 # shellcheck disable=SC2004
 ROUNDED_SIZE=$(((($size+$MB-1)/$MB)*$MB))
 echo "Resizing raw image to $ROUNDED_SIZE"
-qemu-img resize -f raw "$VHDDISK.work" $ROUNDED_SIZE
+qemu-img resize -f raw "$RAWIMAGE" $ROUNDED_SIZE
 echo "Converting $RAWIMAGE to $VHDDISK"
-qemu-img convert -f raw -o subformat=fixed,force_size -O vpc "$VHDDISK.work" "$VHDDISK"
-echo "Done"
-rm -rf "$VHDDISK.work"
+qemu-img convert -f raw -o subformat=fixed,force_size -O vpc "$RAWIMAGE" "$VHDDISK"
+echo "Done"

tools-image/build-arm-image.sh

@@ -100,7 +100,6 @@ usage()
     echo " --local: (optional) Use local repository when building"
     echo " --directory: (optional) A directory which will be used for active/passive/recovery system"
     echo " --model: (optional) The board model"
-    echo " --efi-dir: (optional) A directory with files which will be added to the efi partition"
     exit 1
 }
 
@@ -175,10 +174,6 @@ while [ "$#" -gt 0 ]; do
             shift 1
             model=$1
             ;;
-        --efi-dir)
-            shift 1
-            efi_dir=$1
-            ;;
         --final-repo)
             shift 1
             final_repo=$1
@@ -272,7 +267,7 @@ ensure_dir_structure $TARGET
 # Download the container image
 if [ -z "$directory" ]; then
   echo ">>> Downloading container image"
-  elemental pull-image $( (( $local_build == 'true')) && printf %s '--local' ) $container_image $TARGET
+  elemental pull-image $container_image $TARGET
 else
   echo ">>> Copying files from $directory"
   rsync -axq --exclude='host' --exclude='mnt' --exclude='proc' --exclude='sys' --exclude='dev' --exclude='tmp' ${directory}/ $TARGET
@@ -318,10 +313,6 @@ if [ -z "$EFI" ]; then
 fi
 
 cp -rfv /arm/grub/efi/* $EFI
-if [ -n "$EFI" ] && [  -n "$efi_dir" ]; then
-  echo "Copy $efi_dir to EFI directory"
-  cp -rfv $efi_dir/* $EFI
-fi
 
 echo ">> Writing image and partition table"
 dd if=/dev/zero of="${output_image}" bs=1024000 count="${size}" || exit 1

tools-image/gce.sh

@@ -2,14 +2,15 @@
 
 # Transform a raw image disk to gce compatible
 RAWIMAGE="$1"
-OUT="${2:-$RAWIMAGE.gce.raw}"
-cp -rf $RAWIMAGE $OUT
 
 GB=$((1024*1024*1024))
-size=$(qemu-img info -f raw --output json "$OUT" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
+size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
 # shellcheck disable=SC2004
 ROUNDED_SIZE=$(echo "$size/$GB+1"|bc)
 echo "Resizing raw image from \"$size\"MB to \"$ROUNDED_SIZE\"GB"
-qemu-img resize -f raw "$OUT" "$ROUNDED_SIZE"G
-echo "Compressing raw image $OUT to $OUT.tar.gz"
-tar -c -z --format=oldgnu -f "$OUT".tar.gz $OUT
+qemu-img resize -f raw "$RAWIMAGE" "$ROUNDED_SIZE"G
+echo "Compressing raw image $RAWIMAGE to $RAWIMAGE.tar.gz"
+tar -c -z --format=oldgnu -f "$RAWIMAGE".tar.gz $RAWIMAGE
+echo "Restoring size to original raw image"
+qemu-img resize -f raw "$RAWIMAGE" --shrink "$size"
+echo "Done"

tools-image/netboot.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-# Extracts squashfs, kernel, initrd and generates a ipxe template script
-
-ISO=$1
-OUTPUT_NAME=$2
-ARTIFACT_NAME=$(basename $OUTPUT_NAME)
-
-isoinfo -x /rootfs.squashfs -R -i $ISO > $OUTPUT_NAME.squashfs
-isoinfo -x /boot/kernel -R -i $ISO > $OUTPUT_NAME-kernel
-isoinfo -x /boot/initrd -R -i $ISO > $OUTPUT_NAME-initrd
-
-URL=${URL:-https://github.com/kairos-io/kairos/releases/download}
-
-cat > $OUTPUT_NAME.ipxe << EOF
-#!ipxe
-set url ${URL}/
-set kernel $ARTIFACT_NAME-kernel
-set initrd $ARTIFACT_NAME-initrd
-set rootfs $ARTIFACT_NAME.squashfs
-# set config https://example.com/machine-config
-# set cmdline extra.values=1
-kernel \${url}/\${kernel} initrd=\${initrd} ip=dhcp rd.cos.disable root=live:\${url}/\${rootfs} netboot nodepair.enable config_url=\${config} console=tty1 console=ttyS0 \${cmdline}
-initrd \${url}/\${initrd}
-boot
-EOF

tools-image/raw-images.sh

@@ -1,15 +1,10 @@
 #!/bin/bash
 # Generates EFI bootable images (statically)
-# This is a re-adaptation of https://github.com/rancher/elemental-toolkit/blob/v0.8.10-1/images/img-builder.sh, which was dropped
-# How to use:
-# First extract the image which you want to create an image from:
-### luet util unpack <image> rootfs
-# Then convert it to a raw disk (EFI only):
-### docker run -v $PWD:/output --entrypoint /raw-images.sh -ti --rm test-image /output/rootfs /output/foo.raw cloud-init.yaml
+# luet util unpack <image> rootfs
+# docker run -v $PWD:/output --entrypoint /raw-images.sh -ti --rm test-image /output/rootfs /output/foo.raw cloud-init.yaml
 
 : "${OEM_LABEL:=COS_OEM}"
 : "${RECOVERY_LABEL:=COS_RECOVERY}"
-: "${EXTEND:=}"
 
 DIRECTORY=$1
 OUT=${2:-disk.raw}
@@ -31,8 +26,6 @@ echo "Generating squashfs from $DIRECTORY"
 mksquashfs $DIRECTORY recovery.squashfs -b 1024k -comp xz -Xbcj x86
 mv recovery.squashfs /build/root/cOS/recovery.squashfs
 
-grub2-editenv /build/root/grub_oem_env set "default_menu_entry=Kairos"
-
 # Create a 2GB filesystem for RECOVERY including the contents for root (grub config and squasfs container)
 truncate -s $((2048*1024*1024)) rootfs.part
 mkfs.ext2 -L "${RECOVERY_LABEL}" -d /build/root rootfs.part
@@ -67,11 +60,6 @@ truncate -s $((3*1024*1024)) $OUT
 # Add an extra MB at the end of the disk for the gpt headers, in fact 34 sectors would be enough, but adding some more does not hurt.
 truncate -s "+$((1024*1024))" $OUT
 
-if [ -n "$EXTEND" ]; then
-  echo "Extending image of $EXTEND MB"
-  truncate -s "+$(($EXTEND*1024*1024))" $OUT
-fi
-
 # Create the partition table in $OUT (assumes sectors of 512 bytes)
 sgdisk -n 1:2048:+2M -c 1:legacy -t 1:EF02 $OUT
 sgdisk -n 2:0:+20M -c 2:UEFI -t 2:EF00 $OUT