woodpecker/pipeline/backend/kubernetes/pod.go

// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
	"context"
	"fmt"
	"maps"
	"strings"

	"github.com/rs/zerolog/log"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/common"
	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
)

const (
	StepLabel = "step"
)

func mkPod(namespace, name, image, workDir, goos, serviceAccountName string,
	pool, privileged bool,
	commands, vols, pullSecretNames []string,
	labels, annotations, env, nodeSelector map[string]string,
	extraHosts []types.HostAlias, tolerations []types.Toleration, resources types.Resources,
	securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (*v1.Pod, error) {
	var err error

	meta := podMeta(name, namespace, labels, annotations)

	spec, err := podSpec(serviceAccountName, vols, pullSecretNames, env, nodeSelector, extraHosts, tolerations,
		securityContext, securityContextConfig)
	if err != nil {
		return nil, err
	}

	container, err := podContainer(name, image, workDir, goos, pool, privileged, commands, vols, env,
		resources, securityContext)
	if err != nil {
		return nil, err
	}
	spec.Containers = append(spec.Containers, container)

	pod := &v1.Pod{
		ObjectMeta: meta,
		Spec:       spec,
	}

	return pod, nil
}

func stepToPodName(step *types.Step) (name string, err error) {
	if step.Type == types.StepTypeService {
		return serviceName(step)
	}
	return podName(step)
}

func podName(step *types.Step) (string, error) {
	return dnsName(step.Name)
}

func podMeta(name, namespace string, labels, annotations map[string]string) metav1.ObjectMeta {
	meta := metav1.ObjectMeta{
		Name:        name,
		Namespace:   namespace,
		Annotations: annotations,
	}

	if labels == nil {
		labels = make(map[string]string, 1)
	}
	labels[StepLabel] = name
	meta.Labels = labels

	return meta
}

func podSpec(serviceAccountName string, vols, pullSecretNames []string, env, backendNodeSelector map[string]string,
	extraHosts []types.HostAlias, backendTolerations []types.Toleration,
	securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (v1.PodSpec, error) {
	var err error
	spec := v1.PodSpec{
		RestartPolicy:      v1.RestartPolicyNever,
		ServiceAccountName: serviceAccountName,
		ImagePullSecrets:   imagePullSecretsReferences(pullSecretNames),
	}

	spec.HostAliases = hostAliases(extraHosts)
	spec.NodeSelector = nodeSelector(backendNodeSelector, env["CI_SYSTEM_PLATFORM"])
	spec.Tolerations = tolerations(backendTolerations)
	spec.SecurityContext = podSecurityContext(securityContext, securityContextConfig)
	spec.Volumes, err = volumes(vols)
	if err != nil {
		return spec, err
	}

	return spec, nil
}

func podContainer(name, image, workDir, goos string, pull, privileged bool, commands, volumes []string, env map[string]string, resources types.Resources,
	securityContext *types.SecurityContext,
) (v1.Container, error) {
	var err error
	container := v1.Container{
		Name:       name,
		Image:      image,
		WorkingDir: workDir,
	}

	if pull {
		container.ImagePullPolicy = v1.PullAlways
	}

	if len(commands) != 0 {
		scriptEnv, command, args := common.GenerateContainerConf(commands, goos)
		container.Command = command
		container.Args = args
		maps.Copy(env, scriptEnv)
	}

	container.Env = mapToEnvVars(env)
	container.SecurityContext = containerSecurityContext(securityContext, privileged)

	container.Resources, err = resourceRequirements(resources)
	if err != nil {
		return container, err
	}

	container.VolumeMounts, err = volumeMounts(volumes)
	if err != nil {
		return container, err
	}

	return container, nil
}

func volumes(volumes []string) ([]v1.Volume, error) {
	var vols []v1.Volume

	for _, v := range volumes {
		volumeName, err := volumeName(v)
		if err != nil {
			return nil, err
		}
		vols = append(vols, volume(volumeName))
	}

	return vols, nil
}

func volume(name string) v1.Volume {
	pvcSource := v1.PersistentVolumeClaimVolumeSource{
		ClaimName: name,
		ReadOnly:  false,
	}
	return v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			PersistentVolumeClaim: &pvcSource,
		},
	}
}

func volumeMounts(volumes []string) ([]v1.VolumeMount, error) {
	var mounts []v1.VolumeMount

	for _, v := range volumes {
		volumeName, err := volumeName(v)
		if err != nil {
			return nil, err
		}

		mount := volumeMount(volumeName, volumeMountPath(v))
		mounts = append(mounts, mount)
	}
	return mounts, nil
}

func volumeMount(name, path string) v1.VolumeMount {
	return v1.VolumeMount{
		Name:      name,
		MountPath: path,
	}
}

// Here is the service IPs (placed in /etc/hosts in the Pod)
func hostAliases(extraHosts []types.HostAlias) []v1.HostAlias {
	hostAliases := []v1.HostAlias{}
	for _, extraHost := range extraHosts {
		hostAlias := hostAlias(extraHost)
		hostAliases = append(hostAliases, hostAlias)
	}
	return hostAliases
}

func hostAlias(extraHost types.HostAlias) v1.HostAlias {
	return v1.HostAlias{
		IP:        extraHost.IP,
		Hostnames: []string{extraHost.Name},
	}
}

func imagePullSecretsReferences(imagePullSecretNames []string) []v1.LocalObjectReference {
	log.Trace().Msgf("Using the image pull secrets: %v", imagePullSecretNames)

	secretReferences := make([]v1.LocalObjectReference, len(imagePullSecretNames))
	for i, imagePullSecretName := range imagePullSecretNames {
		secretReferences[i] = imagePullSecretsReference(imagePullSecretName)
	}
	return secretReferences
}

func imagePullSecretsReference(imagePullSecretName string) v1.LocalObjectReference {
	return v1.LocalObjectReference{
		Name: imagePullSecretName,
	}
}

func resourceRequirements(resources types.Resources) (v1.ResourceRequirements, error) {
	var err error
	requirements := v1.ResourceRequirements{}

	requirements.Requests, err = resourceList(resources.Requests)
	if err != nil {
		return requirements, err
	}

	requirements.Limits, err = resourceList(resources.Limits)
	if err != nil {
		return requirements, err
	}

	return requirements, nil
}

func resourceList(resources map[string]string) (v1.ResourceList, error) {
	requestResources := v1.ResourceList{}
	for key, val := range resources {
		resName := v1.ResourceName(key)
		resVal, err := resource.ParseQuantity(val)
		if err != nil {
			return nil, fmt.Errorf("resource request '%v' quantity '%v': %w", key, val, err)
		}
		requestResources[resName] = resVal
	}
	return requestResources, nil
}

func nodeSelector(backendNodeSelector map[string]string, platform string) map[string]string {
	nodeSelector := make(map[string]string)

	if platform != "" {
		arch := strings.Split(platform, "/")[1]
		nodeSelector[v1.LabelArchStable] = arch
		log.Trace().Msgf("Using the node selector from the Agent's platform: %v", nodeSelector)
	}

	if len(backendNodeSelector) > 0 {
		log.Trace().Msgf("Appending labels to the node selector from the backend options: %v", backendNodeSelector)
		maps.Copy(nodeSelector, backendNodeSelector)
	}

	return nodeSelector
}

func tolerations(backendTolerations []types.Toleration) []v1.Toleration {
	var tolerations []v1.Toleration

	if len(backendTolerations) > 0 {
		log.Trace().Msgf("Tolerations that will be used in the backend options: %v", backendTolerations)
		for _, backendToleration := range backendTolerations {
			toleration := toleration(backendToleration)
			tolerations = append(tolerations, toleration)
		}
	}

	return tolerations
}

func toleration(backendToleration types.Toleration) v1.Toleration {
	return v1.Toleration{
		Key:               backendToleration.Key,
		Operator:          v1.TolerationOperator(backendToleration.Operator),
		Value:             backendToleration.Value,
		Effect:            v1.TaintEffect(backendToleration.Effect),
		TolerationSeconds: backendToleration.TolerationSeconds,
	}
}

func podSecurityContext(sc *types.SecurityContext, secCtxConf SecurityContextConfig) *v1.PodSecurityContext {
	var (
		nonRoot *bool
		user    *int64
		group   *int64
		fsGroup *int64
	)

	if sc != nil && sc.RunAsNonRoot != nil {
		if *sc.RunAsNonRoot {
			nonRoot = sc.RunAsNonRoot // true
		}
	} else if secCtxConf.RunAsNonRoot {
		nonRoot = &secCtxConf.RunAsNonRoot // true
	}

	if sc != nil {
		user = sc.RunAsUser
		group = sc.RunAsGroup
		fsGroup = sc.FSGroup
	}

	if nonRoot == nil && user == nil && group == nil && fsGroup == nil {
		return nil
	}

	securityContext := &v1.PodSecurityContext{
		RunAsNonRoot: nonRoot,
		RunAsUser:    user,
		RunAsGroup:   group,
		FSGroup:      fsGroup,
	}
	log.Trace().Msgf("Pod security context that will be used: %v", securityContext)
	return securityContext
}

func containerSecurityContext(sc *types.SecurityContext, stepPrivileged bool) *v1.SecurityContext {
	var privileged *bool

	if sc != nil && sc.Privileged != nil && *sc.Privileged {
		privileged = sc.Privileged // true
	} else if stepPrivileged {
		privileged = &stepPrivileged // true
	}

	if privileged == nil {
		return nil
	}

	securityContext := &v1.SecurityContext{
		Privileged: privileged,
	}
	log.Trace().Msgf("Container security context that will be used: %v", securityContext)
	return securityContext
}

func mapToEnvVars(m map[string]string) []v1.EnvVar {
	var ev []v1.EnvVar
	for k, v := range m {
		ev = append(ev, v1.EnvVar{
			Name:  k,
			Value: v,
		})
	}
	return ev
}

func startPod(ctx context.Context, engine *kube, step *types.Step) (*v1.Pod, error) {
	podName, err := podName(step)
	if err != nil {
		return nil, err
	}

	pod, err := mkPod(engine.config.Namespace, podName, step.Image, step.WorkingDir, engine.goos, step.BackendOptions.Kubernetes.ServiceAccountName,
		step.Pull, step.Privileged,
		step.Commands, step.Volumes, engine.config.ImagePullSecretNames,
		engine.config.PodLabels, engine.config.PodAnnotations, step.Environment, step.BackendOptions.Kubernetes.NodeSelector,
		step.ExtraHosts, step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext)
	if err != nil {
		return nil, err
	}

	log.Trace().Msgf("Creating pod: %s", pod.Name)
	return engine.client.CoreV1().Pods(engine.config.Namespace).Create(ctx, pod, metav1.CreateOptions{})
}

func stopPod(ctx context.Context, engine *kube, step *types.Step, deleteOpts metav1.DeleteOptions) error {
	podName, err := podName(step)
	if err != nil {
		return err
	}
	log.Trace().Str("name", podName).Msg("Deleting pod")

	err = engine.client.CoreV1().Pods(engine.config.Namespace).Delete(ctx, podName, deleteOpts)
	if errors.IsNotFound(err) {
		// Don't abort on 404 errors from k8s, they most likely mean that the pod hasn't been created yet, usually because pipeline was canceled before running all steps.
		return nil
	}
	return err
}