Add scripts to generate raw images

See: https://github.com/kairos-io/kairos/issues/377

controllers/deployment.go

@@ -116,6 +116,45 @@ func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact)
 	privileged := false
 	serviceAccount := false
 
+	cmd := fmt.Sprintf(
+		"/entrypoint.sh --debug --name %s build-iso --date=false --output /public dir:/rootfs",
+		artifact.Name,
+	)
+
+	volumeMounts := []v1.VolumeMount{
+		{
+			Name:      "public",
+			MountPath: "/public",
+		},
+		{
+			Name:      "rootfs",
+			MountPath: "/rootfs",
+		},
+	}
+
+	if artifact.Spec.GRUBConfig != "" {
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      "config",
+			MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
+			SubPath:   "grub.cfg",
+		})
+	}
+
+	if artifact.Spec.CloudConfig != "" {
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      "config",
+			MountPath: "/iso/iso-overlay/cloud_config.yaml",
+			SubPath:   "config",
+		})
+	}
+
+	if artifact.Spec.CloudConfig != "" || artifact.Spec.GRUBConfig != "" {
+		cmd = fmt.Sprintf(
+			"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /public dir:/rootfs",
+			artifact.Name,
+		)
+	}
+
 	buildIsoContainer := v1.Container{
 		ImagePullPolicy: v1.PullAlways,
 		SecurityContext: &v1.SecurityContext{Privileged: &privileged},
@@ -123,31 +162,9 @@ func (r *OSArtifactReconciler) genDeployment(artifact buildv1alpha1.OSArtifact)
 		Image:           r.ToolImage,
 		Command:         []string{"/bin/bash", "-cxe"},
 		Args: []string{
-			fmt.Sprintf(
-				"/entrypoint.sh --debug --name %s build-iso --date=false --overlay-iso /iso/iso-overlay --output /public dir:/rootfs",
-				artifact.Name,
-			),
-		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      "public",
-				MountPath: "/public",
-			},
-			{
-				Name:      "config",
-				MountPath: "/iso/iso-overlay/cloud_config.yaml",
-				SubPath:   "config",
-			},
-			{
-				Name:      "config",
-				MountPath: "/iso/iso-overlay/boot/grub2/grub.cfg",
-				SubPath:   "grub.cfg",
-			},
-			{
-				Name:      "rootfs",
-				MountPath: "/rootfs",
-			},
+			cmd,
 		},
+		VolumeMounts: volumeMounts,
 	}
 
 	servingContainer := v1.Container{

tools-image/Dockerfile

@@ -1,7 +1,7 @@
-ARG ELEMENTAL_CLI_VERSION=0.20220921
+ARG ELEMENTAL_CLI_VERSION=0.20221020.3
 ARG LEAP_VERSION=15.4
-ARG LUET_VERSION=0.32.5
-FROM quay.io/kairos/packages:elemental-cli-system-0.20220921 AS elemental
+ARG LUET_VERSION=0.33.0
+FROM quay.io/kairos/packages:elemental-cli-system-$ELEMENTAL_CLI_VERSION AS elemental
 FROM quay.io/luet/base:$LUET_VERSION AS luet
 
 ### TODO: Replace those naked Dockerfiles copies with luet install so we can keep track of all versioning with 1 repository tag
@@ -20,11 +20,19 @@ FROM quay.io/kairos/packages:raspberrypi-firmware-firmware-2021.03.10-2.1 AS rpi
 FROM quay.io/kairos/packages:raspberrypi-firmware-config-firmware-2021.03.10-2.1 AS rpi-firmware-config
 FROM quay.io/kairos/packages:raspberrypi-firmware-dt-firmware-2021.03.15-2.1 AS rpi-firmware-dt
 
+## PineBook64 Pro
+FROM quay.io/kairos/packages:u-boot-rockchip-arm-vendor-blob-0.1 AS pinebook-u-boot
+
+## Generic ARM artifacts
 FROM quay.io/kairos/packages-arm64:grub-efi-static-0.1  AS grub-efi
 FROM quay.io/kairos/packages-arm64:grub-config-static-0.1  AS grub-config
 FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.1   AS grub-artifacts
 
- 
+ ## RAW images
+FROM quay.io/kairos/packages:grub-efi-static-0.1  AS grub-raw-efi
+FROM quay.io/kairos/packages:grub-config-static-0.1  AS grub-raw-config
+FROM quay.io/kairos/packages:grub-artifacts-static-0.1   AS grub-raw-artifacts
+
 FROM opensuse/leap:$LEAP_VERSION
 COPY --from=elemental /usr/bin/elemental /usr/bin/elemental
 COPY --from=luet /usr/bin/luet /usr/bin/luet
@@ -33,14 +41,24 @@ COPY --from=luet /usr/bin/luet /usr/bin/luet
 COPY --from=grub2 / /grub2
 COPY --from=efi / /efi
 
+# RAW images
+COPY --from=grub-raw-efi / /raw/grub
+COPY --from=grub-raw-config / /raw/grubconfig
+COPY --from=grub-raw-artifacts / /raw/grubartifacts
+
 # RPI64
 COPY --from=rpi-u-boot / /rpi/u-boot
 COPY --from=rpi-firmware / /rpi/rpi-firmware
 COPY --from=rpi-firmware-config / /rpi/rpi-firmware-config
 COPY --from=rpi-firmware-dt / /rpi/rpi-firmware-dt
-COPY --from=grub-efi / /rpi64/grub/efi
-COPY --from=grub-config / /rpi64/grub/config
-COPY --from=grub-artifacts / /rpi64/grub/artifacts
+
+# Pinebook
+COPY --from=pinebook-u-boot / /pinebookpro/u-boot
+
+# Generic
+COPY --from=grub-efi / /arm/grub/efi
+COPY --from=grub-config / /arm/grub/config
+COPY --from=grub-artifacts / /arm/grub/artifacts
 
 RUN zypper ref && zypper dup -y
 
@@ -62,5 +80,12 @@ COPY ./add-cloud-init.sh /add-cloud-init.sh
 COPY ./build-arm-image.sh /build-arm-image.sh
 COPY ./arm /arm
 
+# RAW images helpers
+COPY ./gce.sh /gce.sh
+COPY ./raw-images.sh /raw-images.sh
+COPY ./azure.sh /azure.sh
+
+COPY defaults.yaml /defaults.yaml
+
 
 ENTRYPOINT [ "/entrypoint.sh" ]

tools-image/arm/boards/pinebookpro.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+image=$1
+
+if [ -z "$image" ]; then
+    echo "No image specified"
+    exit 1
+fi
+
+LOADER_OFFSET=${LOADER_OFFSET:-"64"}
+LOADER_IMAGE=${LOADER_IMAGE:-"idbloader.img"}
+UBOOT_IMAGE=${UBOOT_IMAGE:-"u-boot.itb"}
+UBOOT_OFFSET=${UBOOT_OFFSET:-"16384"}
+
+echo "Writing idbloader"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${LOADER_IMAGE} of="$image" conv=fsync seek=${LOADER_OFFSET}
+echo "Writing u-boot image"
+dd conv=notrunc if=/pinebookpro/u-boot/usr/lib/u-boot/pinebook-pro-rk3399/${UBOOT_IMAGE} of="$image" conv=fsync seek=${UBOOT_OFFSET}
+sync $image

tools-image/azure.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Transform a raw image disk to azure vhd
+RAWIMAGE="$1"
+VHDDISK="${2:-disk.vhd}"
+
+MB=$((1024*1024))
+size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
+# shellcheck disable=SC2004
+ROUNDED_SIZE=$(((($size+$MB-1)/$MB)*$MB))
+echo "Resizing raw image to $ROUNDED_SIZE"
+qemu-img resize -f raw "$RAWIMAGE" $ROUNDED_SIZE
+echo "Converting $RAWIMAGE to $VHDDISK"
+qemu-img convert -f raw -o subformat=fixed,force_size -O vpc "$RAWIMAGE" "$VHDDISK"
+echo "Done"

tools-image/build-arm-image.sh

@@ -298,9 +298,9 @@ cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img
 tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img
 
 # Install real grub config to recovery
-cp -rfv /$model/grub/config/* $RECOVERY
+cp -rfv /arm/grub/config/* $RECOVERY
 mkdir -p $RECOVERY/grub2
-cp -rfv /$model/grub/artifacts/* $RECOVERY/grub2
+cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
 
 sync
 
@@ -312,7 +312,7 @@ if [ -z "$EFI" ]; then
   exit 1
 fi
 
-cp -rfv /$model/grub/efi/* $EFI
+cp -rfv /arm/grub/efi/* $EFI
 
 echo ">> Writing image and partition table"
 dd if=/dev/zero of="${output_image}" bs=1024000 count="${size}" || exit 1
@@ -374,15 +374,20 @@ mount $recovery $WORKDIR/recovery
 mount $state $WORKDIR/state
 mount $efi $WORKDIR/efi
 
+mkdir $WORKDIR/persistent
+mount $persistent $WORKDIR/persistent
+mkdir -p $WORKDIR/persistent/cloud-config
+
+cp -rfv /defaults.yaml $WORKDIR/persistent/cloud-config/01_defaults.yaml
+
+grub2-editenv $WORKDIR/state/grub_oem_env set "default_menu_entry=Kairos"
+
 # Set a OEM config file if specified
 if [ -n "$config" ]; then
   echo ">> Copying $config OEM config file"
-  mkdir $WORKDIR/persistent
-  mount $persistent $WORKDIR/persistent
-  mkdir $WORKDIR/persistent/cloud-config
   get_url $config $WORKDIR/persistent/cloud-config/99_custom.yaml
-  umount $WORKDIR/persistent
 fi
+umount $WORKDIR/persistent
 
 # Copy over content
 cp -arf $EFI/* $WORKDIR/efi

tools-image/defaults.yaml

@@ -0,0 +1,8 @@
+
+name: "Default user"
+stages:
+   initramfs:
+     - name: "Set default user/pass"
+       users:
+        kairos:
+          passwd: "kairos"

tools-image/gce.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Transform a raw image disk to gce compatible
+RAWIMAGE="$1"
+
+GB=$((1024*1024*1024))
+size=$(qemu-img info -f raw --output json "$RAWIMAGE" | gawk 'match($0, /"virtual-size": ([0-9]+),/, val) {print val[1]}')
+# shellcheck disable=SC2004
+ROUNDED_SIZE=$(echo "$size/$GB+1"|bc)
+echo "Resizing raw image from \"$size\"MB to \"$ROUNDED_SIZE\"GB"
+qemu-img resize -f raw "$RAWIMAGE" "$ROUNDED_SIZE"G
+echo "Compressing raw image $RAWIMAGE to $RAWIMAGE.tar.gz"
+tar -c -z --format=oldgnu -f "$RAWIMAGE".tar.gz $RAWIMAGE
+echo "Restoring size to original raw image"
+qemu-img resize -f raw "$RAWIMAGE" --shrink "$size"
+echo "Done"

tools-image/raw-images.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Generates EFI bootable images (statically)
+# luet util unpack <image> rootfs
+# docker run -v $PWD:/output --entrypoint /raw-images.sh -ti --rm test-image /output/rootfs /output/foo.raw cloud-init.yaml
+
+: "${OEM_LABEL:=COS_OEM}"
+: "${RECOVERY_LABEL:=COS_RECOVERY}"
+
+DIRECTORY=$1
+OUT=${2:-disk.raw}
+CONFIG=$3
+
+echo "Output: $OUT"
+
+set -e
+
+mkdir -p /build/root/grub2
+mkdir /build/root/cOS
+mkdir /build/efi
+
+cp -rf /raw/grub/* /build/efi
+cp -rf /raw/grubconfig/* /build/root
+cp -rf /raw/grubartifacts/* /build/root/grub2
+
+echo "Generating squashfs from $DIRECTORY"
+mksquashfs $DIRECTORY recovery.squashfs -b 1024k -comp xz -Xbcj x86
+mv recovery.squashfs /build/root/cOS/recovery.squashfs
+
+# Create a 2GB filesystem for RECOVERY including the contents for root (grub config and squasfs container)
+truncate -s $((2048*1024*1024)) rootfs.part
+mkfs.ext2 -L "${RECOVERY_LABEL}" -d /build/root rootfs.part
+
+# Create the EFI partition FAT16 and include the EFI image and a basic grub.cfg
+truncate -s $((20*1024*1024)) efi.part
+
+mkfs.fat -F16 -n COS_GRUB efi.part
+mcopy -s -i efi.part /build/efi/EFI ::EFI
+
+# Create the grubenv forcing first boot to be on recovery system
+mkdir -p /build/oem
+cp /build/root/etc/cos/grubenv_firstboot /build/oem/grubenv
+if [ -n "$CONFIG" ]; then
+  echo "Copying config file ($CONFIG)"
+  cp $CONFIG /build/oem
+fi
+
+# Create a 64MB filesystem for OEM volume
+truncate -s $((64*1024*1024)) oem.part
+mkfs.ext2 -L "${OEM_LABEL}" -d /build/oem oem.part
+
+echo "Generating image $OUT"
+# Create disk image, add 3MB of initial free space to disk, 1MB is for proper alignement, 2MB are for the hybrid legacy boot.
+truncate -s $((3*1024*1024)) $OUT
+{
+    cat efi.part
+    cat oem.part
+    cat rootfs.part
+} >> $OUT
+
+# Add an extra MB at the end of the disk for the gpt headers, in fact 34 sectors would be enough, but adding some more does not hurt.
+truncate -s "+$((1024*1024))" $OUT
+
+# Create the partition table in $OUT (assumes sectors of 512 bytes)
+sgdisk -n 1:2048:+2M -c 1:legacy -t 1:EF02 $OUT
+sgdisk -n 2:0:+20M -c 2:UEFI -t 2:EF00 $OUT
+sgdisk -n 3:0:+64M -c 3:oem -t 3:8300 $OUT
+sgdisk -n 4:0:+2048M -c 4:root -t 4:8300 $OUT