Multi-arch images for apparmor-loader container

Originally from:
https://github.com/kubernetes/contrib/tree/master/apparmor/loader

Moving the code here to prevent bit-rot and to be sure we can recreate
or update the images on demand. Moving it here also ensures we can use
the common harness to build the multi-arch manifests needed for running
the apparmor e2e test can run on multiple architectures.

Change-Id: Idece17c494fc944c0aaef64805d2f0e3c4d7fb28

test/images/BUILD

@@ -11,6 +11,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//test/images/apparmor-loader:all-srcs",
         "//test/images/entrypoint-tester:all-srcs",
         "//test/images/fakegitserver:all-srcs",
         "//test/images/liveness:all-srcs",

test/images/apparmor-loader/BASEIMAGE

@@ -0,0 +1,4 @@
+amd64=alpine:3.8
+arm=arm32v6/alpine:3.8
+arm64=arm64v8/alpine:3.8
+ppc64le=ppc64le/alpine:3.8

test/images/apparmor-loader/BUILD

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["loader.go"],
+    importpath = "k8s.io/kubernetes/test/images/apparmor-loader",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/golang/glog:go_default_library"],
+)
+
+go_binary(
+    name = "apparmor-loader",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/images/apparmor-loader/Dockerfile

@@ -0,0 +1,26 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
+
+RUN apk add apparmor libapparmor --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ --allow-untrusted
+
+ADD loader /usr/bin/loader
+
+ENTRYPOINT ["/usr/bin/loader", "-logtostderr", "-v=2"]
+
+# Default directory to watch.
+CMD ["/profiles"]

test/images/apparmor-loader/Makefile

@@ -0,0 +1,25 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SRCS=loader
+ARCH ?= amd64
+TARGET ?= $(CURDIR)
+GOLANG_VERSION ?= latest
+SRC_DIR = $(notdir $(shell pwd))
+export
+
+bin:
+	../image-util.sh bin $(SRCS)
+
+.PHONY: bin

test/images/apparmor-loader/README.md

@@ -0,0 +1,66 @@
+# AppArmor Profile Loader
+
+This is a small proof-of-concept daemon to demonstrate how AppArmor profiles can be loaded onto
+nodes of a Kubernetes cluster. It is not considered production ready, nor will it be supported as a
+long-term solution.
+
+## Running the AppArmor Profile Loader
+
+The [example-daemon.yaml](example-daemon.yaml) provides an example manifest for running the loader
+as a cluster DaemonSet. In this example, the loader runs in a DaemonSet pod on each node in the
+cluster, and periodically (every 30 seconds) polls for new profiles in the `apparmor-profiles`
+configmap ([example manifest](example-configmap.yaml)). It is recommended to run the Daemon and
+ConfigMap in a separate, restricted namespace:
+
+    $ kubectl create -f example-namespace.yaml
+    $ kubectl create -f example-configmap.yaml # Includes the k8s-nginx profile
+    $ kubectl create -f example-daemon.yaml
+
+Check that the profile was loaded:
+
+    $ POD=$(kubectl --namespace apparmor get pod -o jsonpath="{.items[0].metadata.name}")
+    $ kubectl --namespace apparmor logs $POD
+    I0829 22:48:24.917263       1 loader.go:139] Polling /profiles every 30s
+    I0829 22:48:24.954295       1 loader.go:196] Loading profiles from /profiles/k8s-nginx:
+    Addition succeeded for "k8s-nginx".
+    I0829 22:48:24.954328       1 loader.go:100] Successfully loaded profiles: [k8s-nginx]
+
+Trying running a pod with the loaded profile (requires Kubernetes >= v1.4):
+
+    $ kubectl create -f example-pod.yaml
+    # Verify that it's running with the new profile:
+    $ kubectl exec nginx-apparmor cat /proc/1/attr/current
+    k8s-nginx (enforce)
+    $ kubectl exec nginx-apparmor touch /tmp/foo
+    touch: cannot touch '/tmp/foo': Permission denied
+    error: error executing remote command: command terminated with non-zero exit code: Error executing in Docker Container: 1
+
+
+### Standalone
+
+The loader go binary can also be run as a standalone binary on the host. It must be run with root
+privileges:
+
+    sudo loader -logtostderr /path/to/profile/dir
+
+Alternatively, it can be run with the supplied loader docker image:
+
+    PROFILES_PATH=/path/to/profile/dir
+    sudo docker run \
+        --privileged \
+        --detach=true \
+        --volume=/sys:/sys:ro \
+        --volume=/etc/apparmor.d:/etc/apparmor.d:ro \
+        --volume=$PROFILES_PATH:/profiles:ro \
+        --name=aa-loader \
+        google/apparmor-loader:latest
+
+## Build the loader
+
+The loader binary is a simple go program, and can be built with `make all-push WHAT=apparmor-loader`
+(from test/images).
+
+## Limitations
+
+The loader will not unload profiles that are removed, and will not update profiles that are changed.
+This is by design, since there are nuanced issues with changing profiles that are in use.

test/images/apparmor-loader/VERSION

@@ -0,0 +1 @@
+1.0

test/images/apparmor-loader/example-configmap.yaml

@@ -0,0 +1,76 @@
+# An example ConfigMap demonstrating how profiles can be stored as Kubernetes objects, and loaded by
+# the apparmor-loader DaemonSet.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: apparmor-profiles
+  namespace: apparmor
+data:
+  # Filename k8s-nginx maps to the definition of the nginx profile.
+  k8s-nginx: |-
+    #include <tunables/global>
+
+    # From https://github.com/jfrazelle/bane/blob/master/docker-nginx-sample
+    profile k8s-nginx flags=(attach_disconnected,mediate_deleted) {
+      #include <abstractions/base>
+
+      network inet tcp,
+      network inet udp,
+      network inet icmp,
+
+      deny network raw,
+
+      deny network packet,
+
+      file,
+      umount,
+
+      deny /bin/** wl,
+      deny /boot/** wl,
+      deny /dev/** wl,
+      deny /etc/** wl,
+      deny /home/** wl,
+      deny /lib/** wl,
+      deny /lib64/** wl,
+      deny /media/** wl,
+      deny /mnt/** wl,
+      deny /opt/** wl,
+      deny /proc/** wl,
+      deny /root/** wl,
+      deny /sbin/** wl,
+      deny /srv/** wl,
+      deny /tmp/** wl,
+      deny /sys/** wl,
+      deny /usr/** wl,
+
+      audit /** w,
+
+      /var/run/nginx.pid w,
+
+      /usr/sbin/nginx ix,
+
+      deny /bin/dash mrwklx,
+      deny /bin/sh mrwklx,
+      deny /usr/bin/top mrwklx,
+
+      capability chown,
+      capability dac_override,
+      capability setuid,
+      capability setgid,
+      capability net_bind_service,
+
+      deny @{PROC}/{*,**^[0-9*],sys/kernel/shm*} wkx,
+      deny @{PROC}/sysrq-trigger rwklx,
+      deny @{PROC}/mem rwklx,
+      deny @{PROC}/kmem rwklx,
+      deny @{PROC}/kcore rwklx,
+      deny mount,
+      deny /sys/[^f]*/** wklx,
+      deny /sys/f[^s]*/** wklx,
+      deny /sys/fs/[^c]*/** wklx,
+      deny /sys/fs/c[^g]*/** wklx,
+      deny /sys/fs/cg[^r]*/** wklx,
+      deny /sys/firmware/efi/efivars/** rwklx,
+      deny /sys/kernel/security/** rwklx,
+    }

test/images/apparmor-loader/example-daemon.yaml

@@ -0,0 +1,50 @@
+# The example DaemonSet demonstrating how the profile loader can be deployed onto a cluster to
+# automatically load AppArmor profiles from a ConfigMap.
+
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: apparmor-loader
+  # Namespace must match that of the ConfigMap.
+  namespace: apparmor
+spec:
+  template:
+    metadata:
+      name: apparmor-loader
+      labels:
+        daemon: apparmor-loader
+    spec:
+      containers:
+      - name: apparmor-loader
+        image: google/apparmor-loader:latest
+        args:
+          # Tell the loader to pull the /profiles directory every 30 seconds.
+          - -poll
+          - 30s
+          - /profiles
+        securityContext:
+          # The loader requires root permissions to actually load the profiles.
+          privileged: true
+        volumeMounts:
+        - name: sys
+          mountPath: /sys
+          readOnly: true
+        - name: apparmor-includes
+          mountPath: /etc/apparmor.d
+          readOnly: true
+        - name: profiles
+          mountPath: /profiles
+          readOnly: true
+      volumes:
+      # The /sys directory must be mounted to interact with the AppArmor module.
+      - name: sys
+        hostPath:
+          path: /sys
+      # The /etc/apparmor.d directory is required for most apparmor include templates.
+      - name: apparmor-includes
+        hostPath:
+          path: /etc/apparmor.d
+      # Map in the profile data.
+      - name: profiles
+        configMap:
+          name: apparmor-profiles

test/images/apparmor-loader/example-namespace.yaml

@@ -0,0 +1,6 @@
+# The example Namespace used by other example objects.
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apparmor

test/images/apparmor-loader/example-pod.yaml

@@ -0,0 +1,19 @@
+# The example Pod utilizing the profile loaded by the sample daemon.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-apparmor
+  # Note that the Pod does not need to be in the same namespace as the loader.
+  labels:
+    app: nginx
+  annotations:
+    # Tell Kubernetes to apply the AppArmor profile "k8s-nginx".
+    # Note that this is ignored if the Kubernetes node is not running version 1.4 or greater.
+    container.apparmor.security.beta.kubernetes.io/nginx: localhost/k8s-nginx
+spec:
+  containers:
+  - name: nginx
+    image: nginx
+    ports:
+    - containerPort: 80

test/images/apparmor-loader/loader.go

@@ -0,0 +1,260 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+)
+
+var (
+	// The directories to load profiles from.
+	dirs []string
+	poll = flag.Duration("poll", -1, "Poll the directories for new profiles with this interval. Values < 0 disable polling, and exit after loading the profiles.")
+)
+
+const (
+	parser     = "apparmor_parser"
+	apparmorfs = "/sys/kernel/security/apparmor"
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s [FLAG]... [PROFILE_DIR]...\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "Load the AppArmor profiles specified in the PROFILE_DIR directories.\n")
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+
+	dirs = flag.Args()
+	if len(dirs) == 0 {
+		glog.Errorf("Must specify at least one directory.")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// Check that the required parser binary is found.
+	if _, err := exec.LookPath(parser); err != nil {
+		glog.Exitf("Required binary %s not found in PATH", parser)
+	}
+
+	// Check that loaded profiles can be read.
+	if _, err := getLoadedProfiles(); err != nil {
+		glog.Exitf("Unable to access apparmor profiles: %v", err)
+	}
+
+	if *poll < 0 {
+		runOnce()
+	} else {
+		pollForever()
+	}
+}
+
+// No polling: run once and exit.
+func runOnce() {
+	if success, newProfiles := loadNewProfiles(); !success {
+		if len(newProfiles) > 0 {
+			glog.Exitf("Not all profiles were successfully loaded. Loaded: %v", newProfiles)
+		} else {
+			glog.Exit("Error loading profiles.")
+		}
+	} else {
+		if len(newProfiles) > 0 {
+			glog.Infof("Successfully loaded profiles: %v", newProfiles)
+		} else {
+			glog.Warning("No new profiles found.")
+		}
+	}
+}
+
+// Poll the directories indefinitely.
+func pollForever() {
+	glog.V(2).Infof("Polling %s every %s", strings.Join(dirs, ", "), poll.String())
+	pollFn := func() {
+		_, newProfiles := loadNewProfiles()
+		if len(newProfiles) > 0 {
+			glog.V(2).Infof("Successfully loaded profiles: %v", newProfiles)
+		}
+	}
+	pollFn() // Run immediately.
+	ticker := time.NewTicker(*poll)
+	for range ticker.C {
+		pollFn()
+	}
+}
+
+func loadNewProfiles() (success bool, newProfiles []string) {
+	loadedProfiles, err := getLoadedProfiles()
+	if err != nil {
+		glog.Errorf("Error reading loaded profiles: %v", err)
+		return false, nil
+	}
+
+	success = true
+	for _, dir := range dirs {
+		infos, err := ioutil.ReadDir(dir)
+		if err != nil {
+			glog.Warningf("Error reading %s: %v", dir, err)
+			success = false
+			continue
+		}
+
+		for _, info := range infos {
+			path := filepath.Join(dir, info.Name())
+			// If directory, or symlink to a directory, skip it.
+			resolvedInfo, err := resolveSymlink(dir, info)
+			if err != nil {
+				glog.Warningf("Error resolving symlink: %v", err)
+				continue
+			}
+			if resolvedInfo.IsDir() {
+				// Directory listing is shallow.
+				glog.V(4).Infof("Skipping directory %s", path)
+				continue
+			}
+
+			glog.V(4).Infof("Scanning %s for new profiles", path)
+			profiles, err := getProfileNames(path)
+			if err != nil {
+				glog.Warningf("Error reading %s: %v", path, err)
+				success = false
+				continue
+			}
+
+			if unloadedProfiles(loadedProfiles, profiles) {
+				if err := loadProfiles(path); err != nil {
+					glog.Errorf("Could not load profiles: %v", err)
+					success = false
+					continue
+				}
+				// Add new profiles to list of loaded profiles.
+				newProfiles = append(newProfiles, profiles...)
+				for _, profile := range profiles {
+					loadedProfiles[profile] = true
+				}
+			}
+		}
+	}
+
+	return success, newProfiles
+}
+
+func getProfileNames(path string) ([]string, error) {
+	cmd := exec.Command(parser, "--names", path)
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+	out, err := cmd.Output()
+	if err != nil {
+		if stderr.Len() > 0 {
+			glog.Warning(stderr.String())
+		}
+		return nil, fmt.Errorf("error reading profiles from %s: %v", path, err)
+	}
+
+	trimmed := strings.TrimSpace(string(out)) // Remove trailing \n
+	return strings.Split(trimmed, "\n"), nil
+}
+
+func unloadedProfiles(loadedProfiles map[string]bool, profiles []string) bool {
+	for _, profile := range profiles {
+		if !loadedProfiles[profile] {
+			return true
+		}
+	}
+	return false
+}
+
+func loadProfiles(path string) error {
+	cmd := exec.Command(parser, "--verbose", path)
+	stderr := &bytes.Buffer{}
+	cmd.Stderr = stderr
+	out, err := cmd.Output()
+	glog.V(2).Infof("Loading profiles from %s:\n%s", path, out)
+	if err != nil {
+		if stderr.Len() > 0 {
+			glog.Warning(stderr.String())
+		}
+		return fmt.Errorf("error loading profiles from %s: %v", path, err)
+	}
+	return nil
+}
+
+// If the given fileinfo is a symlink, return the FileInfo of the target. Otherwise, return the
+// given fileinfo.
+func resolveSymlink(basePath string, info os.FileInfo) (os.FileInfo, error) {
+	if info.Mode()&os.ModeSymlink == 0 {
+		// Not a symlink.
+		return info, nil
+	}
+	fpath := filepath.Join(basePath, info.Name())
+	resolvedName, err := filepath.EvalSymlinks(fpath)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving symlink %s: %v", fpath, err)
+	}
+	resolvedInfo, err := os.Stat(resolvedName)
+	if err != nil {
+		return nil, fmt.Errorf("error calling stat on %s: %v", resolvedName, err)
+	}
+	return resolvedInfo, nil
+}
+
+// TODO: This is copied from k8s.io/kubernetes/pkg/security/apparmor.getLoadedProfiles.
+//       Refactor that method to expose it in a reusable way, and delete this version.
+func getLoadedProfiles() (map[string]bool, error) {
+	profilesPath := path.Join(apparmorfs, "profiles")
+	profilesFile, err := os.Open(profilesPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open %s: %v", profilesPath, err)
+	}
+	defer profilesFile.Close()
+
+	profiles := map[string]bool{}
+	scanner := bufio.NewScanner(profilesFile)
+	for scanner.Scan() {
+		profileName := parseProfileName(scanner.Text())
+		if profileName == "" {
+			// Unknown line format; skip it.
+			continue
+		}
+		profiles[profileName] = true
+	}
+	return profiles, nil
+}
+
+// The profiles file is formatted with one profile per line, matching a form:
+//   namespace://profile-name (mode)
+//   profile-name (mode)
+// Where mode is {enforce, complain, kill}. The "namespace://" is only included for namespaced
+// profiles. For the purposes of Kubernetes, we consider the namespace part of the profile name.
+func parseProfileName(profileLine string) string {
+	modeIndex := strings.IndexRune(profileLine, '(')
+	if modeIndex < 0 {
+		return ""
+	}
+	return strings.TrimSpace(profileLine[:modeIndex])
+}

test/utils/image/manifest.go

@@ -51,7 +51,7 @@ func (i *ImageConfig) SetVersion(version string) {
 var (
 	AdmissionWebhook         = ImageConfig{e2eRegistry, "webhook", "1.12v2", false}
 	APIServer                = ImageConfig{e2eRegistry, "sample-apiserver", "1.0", false}
-	AppArmorLoader           = ImageConfig{gcRegistry, "apparmor-loader", "0.1", false}
+	AppArmorLoader           = ImageConfig{e2eRegistry, "apparmor-loader", "1.0", false}
 	BusyBox                  = ImageConfig{dockerHubRegistry, "busybox", "1.29", false}
 	CheckMetadataConcealment = ImageConfig{gcRegistry, "check-metadata-concealment", "v0.0.3", false}
 	CudaVectorAdd            = ImageConfig{e2eRegistry, "cuda-vector-add", "1.0", false}