rootfs: agent: Policy support with AGENT_INIT=yes

When building with AGENT_POLICY=yes and AGENT_INIT=yes:
1. Include OPA and the Policy settings in rootfs.
2. Start OPA from the kata agent.

Before these changes, building with both AGENT_POLICY=yes and
AGENT_INIT=yes was unsupported.

Starting OPA from systemd (when AGENT_INIT=no) was already supported.

Fixes: #7615

Signed-off-by: Dan Mihai <dmihai@microsoft.com>

src/agent/src/main.rs

@@ -229,27 +229,6 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
 
     let root_span = span!(tracing::Level::TRACE, "root-span");
 
-    #[cfg(feature = "agent-policy")]
-    {
-        let debug_policy =
-            config.log_level == slog::Level::Debug || config.log_level == slog::Level::Trace;
-        if let Err(e) = AGENT_POLICY
-            .lock()
-            .await
-            .initialize(
-                debug_policy,
-                "http://localhost:8181/v1",
-                "/agent_policy",
-                "/etc/kata-opa/default-policy.rego",
-            )
-            .await
-        {
-            error!(logger, "Failed to initialize agent policy: {:?}", e);
-            // Continuing execution without a security policy could be dangerous.
-            std::process::abort();
-        }
-    }
-
     // XXX: Start the root trace transaction.
     //
     // XXX: Note that *ALL* spans needs to start after this point!!
@@ -355,6 +334,19 @@ async fn start_sandbox(
         s.rtnl.handle_localhost().await?;
     }
 
+    // - When init_mode is true, enabling the localhost link during the
+    //   handle_localhost call above is required before starting OPA with the
+    //   initialize_policy call below.
+    // - When init_mode is false, the Policy could be initialized earlier,
+    //   because initialize_policy doesn't start OPA. OPA is started by
+    //   systemd after localhost has been enabled.
+    #[cfg(feature = "agent-policy")]
+    if let Err(e) = initialize_policy(init_mode).await {
+        error!(logger, "Failed to initialize agent policy: {:?}", e);
+        // Continuing execution without a security policy could be dangerous.
+        std::process::abort();
+    }
+
     let sandbox = Arc::new(Mutex::new(s));
 
     let signal_handler_task = tokio::spawn(setup_signal_handler(
@@ -416,6 +408,18 @@ fn init_agent_as_init(logger: &Logger, unified_cgroup_hierarchy: bool) -> Result
     Ok(())
 }
 
+#[cfg(feature = "agent-policy")]
+async fn initialize_policy(init_mode: bool) -> Result<()> {
+    let opa_addr = "localhost:8181";
+    let agent_policy_path = "/agent_policy";
+    let default_agent_policy = "/etc/kata-opa/default-policy.rego";
+    AGENT_POLICY
+        .lock()
+        .await
+        .initialize(init_mode, opa_addr, agent_policy_path, default_agent_policy)
+        .await
+}
+
 // The Rust standard library had suppressed the default SIGPIPE behavior,
 // see https://github.com/rust-lang/rust/pull/13158.
 // Since the parent's signal handler would be inherited by it's child process,

src/agent/src/policy.rs

@@ -5,6 +5,7 @@
 
 use anyhow::{bail, Result};
 use serde::{Deserialize, Serialize};
+use slog::Drain;
 use tokio::io::AsyncWriteExt;
 use tokio::time::{sleep, Duration};
 
@@ -62,12 +63,12 @@ impl AgentPolicy {
     /// Wait for OPA to start and connect to it.
     pub async fn initialize(
         &mut self,
-        debug_enabled: bool,
+        launch_opa: bool,
-        opa_uri: &str,
+        opa_addr: &str,
         policy_name: &str,
         default_policy: &str,
     ) -> Result<()> {
-        if debug_enabled {
+        if sl!().is_enabled(slog::Level::Debug) {
             self.log_file = Some(
                 tokio::fs::OpenOptions::new()
                     .write(true)
@@ -79,6 +80,11 @@ impl AgentPolicy {
             debug!(sl!(), "policy: log file: {}", POLICY_LOG_FILE);
         }
 
+        if launch_opa {
+            start_opa(opa_addr)?;
+        }
+
+        let opa_uri = format!("http://{opa_addr}/v1");
         self.query_path = format!("{opa_uri}{OPA_DATA_PATH}{policy_name}/");
         self.policy_path = format!("{opa_uri}{OPA_POLICIES_PATH}{policy_name}");
         let opa_client = reqwest::Client::builder().http1_only().build()?;
@@ -102,21 +108,18 @@ impl AgentPolicy {
                 .await
                 .is_ok()
             {
+                self.opa_client = Some(opa_client);
+
                 // Check if requests causing policy errors should actually
                 // be allowed. That is an insecure configuration but is
                 // useful for allowing insecure pods to start, then connect to
                 // them and inspect Guest logs for the root cause of a failure.
-                if let Ok(allow_failures) = self
+                //
+                // Note that post_query returns Ok(false) in case
+                // AllowRequestsFailingPolicy was not defined in the policy.
+                self.allow_failures = self
                     .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
-                    .await
+                    .await?;
-                {
-                    self.allow_failures = allow_failures;
-                } else {
-                    // post_query failed so the the default, secure, value
-                    // allow_failures: false will be used.
-                }
-
-                self.opa_client = Some(opa_client);
                 return Ok(());
             }
         }
@@ -156,6 +159,9 @@ impl AgentPolicy {
             // That is an insecure configuration but is useful for allowing insecure
             // pods to start, then connect to them and inspect Guest logs for the
             // root cause of a failure.
+            //
+            // Note that post_query returns Ok(false) in case
+            // AllowRequestsFailingPolicy was not defined in the policy.
             self.allow_failures = self
                 .post_query("AllowRequestsFailingPolicy", EMPTY_JSON_INPUT)
                 .await?;
@@ -238,3 +244,24 @@ impl AgentPolicy {
         }
     }
 }
+
+fn start_opa(opa_addr: &str) -> Result<()> {
+    let bin_dirs = vec!["/bin", "/usr/bin", "/usr/local/bin"];
+    for bin_dir in &bin_dirs {
+        let opa_path = bin_dir.to_string() + "/opa";
+        if std::fs::metadata(&opa_path).is_ok() {
+            // args copied from kata-opa.service.in.
+            std::process::Command::new(&opa_path)
+                .arg("run")
+                .arg("--server")
+                .arg("--disable-telemetry")
+                .arg("--addr")
+                .arg(opa_addr)
+                .arg("--log-level")
+                .arg("info")
+                .spawn()?;
+            return Ok(());
+        }
+    }
+    bail!("OPA binary not found in {:?}", &bin_dirs);
+}

tools/osbuilder/rootfs-builder/rootfs.sh

@@ -315,9 +315,7 @@ check_env_variables()
 	GOPATH_LOCAL="${GOPATH%%:*}"
 
 	[ "$AGENT_INIT" == "yes" -o "$AGENT_INIT" == "no" ] || die "AGENT_INIT($AGENT_INIT) is invalid (must be yes or no)"
-
 	[ "$AGENT_POLICY" == "yes" -o "$AGENT_POLICY" == "no" ] || die "AGENT_POLICY($AGENT_POLICY) is invalid (must be yes or no)"
-	[ "$AGENT_POLICY" == "no" -o "$AGENT_INIT" == "no" ] || die "AGENT_POLICY($AGENT_POLICY) and AGENT_INIT($AGENT_INIT) is an invalid combination (at least one must be no)"
 
 	[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
 
@@ -678,6 +676,9 @@ EOF
 		install -D -o root -g root -m 0644 "${kata_opa_in_dir}/${policy_file}" -T "${policy_dir}/${policy_file}"
 		ln -sf "${policy_file}" "${policy_dir}/default-policy.rego"
 
+		if [ "${AGENT_INIT}" == "yes" ]; then
+			info "OPA will be started by the kata agent"
+		else
 			# Install the unit file for the kata-opa service.
 			local kata_opa_unit="kata-opa.service"
 			local kata_opa_unit_path="${ROOTFS_DIR}/usr/lib/systemd/system/${kata_opa_unit}"
@@ -693,6 +694,7 @@ EOF
 			mkdir -p "${kata_containers_wants}"
 			ln -sf "${kata_opa_unit_path}" "${kata_containers_wants}/${kata_opa_unit}"
 		fi
+	fi
 
 	info "Check init is installed"
 	[ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"