Merge pull request #1225 from Tim-Zhang/update-cgroup-to-0.2.0

agent: upgrade cgroups to 0.2.0

src/agent/Cargo.lock

@@ -143,13 +143,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
-name = "cgroups"
+name = "cgroups-rs"
-version = "0.1.1-alpha.0"
+version = "0.2.0"
-source = "git+https://github.com/kata-containers/cgroups-rs?branch=stable-0.1.1#8717524f2c95aacd30768b6f0f7d7f2fddef5cac"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02274214de2526e48355facdd16c9d774bba2cf74d135ffb9876a60b4d613464"
 dependencies = [
  "libc",
  "log",
  "nix 0.18.0",
+ "procinfo",
  "regex",
 ]
 
@@ -382,7 +384,7 @@ name = "kata-agent"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "cgroups",
+ "cgroups-rs",
  "lazy_static",
  "libc",
  "log",
@@ -557,6 +559,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nom"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
+
 [[package]]
 name = "num-integer"
 version = "0.1.43"
@@ -693,6 +701,18 @@ dependencies = [
  "libflate",
 ]
 
+[[package]]
+name = "procinfo"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c"
+dependencies = [
+ "byteorder",
+ "libc",
+ "nom",
+ "rustc_version",
+]
+
 [[package]]
 name = "prometheus"
 version = "0.9.0"
@@ -923,13 +943,22 @@ version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
 
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustjail"
 version = "0.1.0"
 dependencies = [
  "anyhow",
  "caps",
- "cgroups",
+ "cgroups-rs",
  "dirs",
  "epoll",
  "lazy_static",
@@ -973,6 +1002,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
 [[package]]
 name = "serde"
 version = "1.0.117"

src/agent/Cargo.toml

@@ -38,7 +38,7 @@ tempfile = "3.1.0"
 prometheus = { version = "0.9.0", features = ["process"] }
 procfs = "0.7.9"
 anyhow = "1.0.32"
-cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
+cgroups = { package = "cgroups-rs", version = "0.2.0" }
 
 [workspace]
 members = [

src/agent/rustjail/Cargo.toml

@@ -24,7 +24,7 @@ regex = "1.1"
 path-absolutize = "1.2.0"
 dirs = "3.0.1"
 anyhow = "1.0.32"
-cgroups = { git = "https://github.com/kata-containers/cgroups-rs", branch = "stable-0.1.1"}
+cgroups = { package = "cgroups-rs", version = "0.2.0" }
 tempfile = "3.1.0"
 epoll = "4.3.1"
 

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -44,35 +44,23 @@ macro_rules! sl {
     };
 }
 
-pub fn load_or_create<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Cgroup<'a> {
+macro_rules! get_controller_or_return_singular_none {
-    let valid_path = path.trim_start_matches('/').to_string();
+    ($cg:ident) => {
-    let cg = load(h.clone(), &valid_path);
+        match $cg.controller_of() {
-    match cg {
+            Some(c) => c,
-        Some(cg) => cg,
+            None => return SingularPtrField::none(),
-        None => {
-            info!(sl!(), "create new cgroup: {}", &valid_path);
-            cgroups::Cgroup::new(h, valid_path.as_str())
-        }
-    }
-}
-
-pub fn load<'a>(h: Box<&'a dyn cgroups::Hierarchy>, path: &str) -> Option<Cgroup<'a>> {
-    let valid_path = path.trim_start_matches('/').to_string();
-    let cg = cgroups::Cgroup::load(h, valid_path.as_str());
-    let cpu_controller: &CpuController = cg.controller_of().unwrap();
-    if cpu_controller.exists() {
-        Some(cg)
-    } else {
-        None
         }
+    };
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Manager {
     pub paths: HashMap<String, String>,
     pub mounts: HashMap<String, String>,
-    // pub rels: HashMap<String, String>,
     pub cpath: String,
+    #[serde(skip)]
+    cgroup: cgroups::Cgroup,
+    relative_paths: HashMap<String, String>,
 }
 
 // set_resource is used to set reources by cgroup controller.
@@ -87,17 +75,11 @@ macro_rules! set_resource {
 
 impl CgroupManager for Manager {
     fn apply(&self, pid: pid_t) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
+        self.cgroup.add_task(CgroupPid::from(pid as u64))?;
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        cg.add_task(CgroupPid::from(pid as u64))?;
         Ok(())
     }
 
     fn set(&self, r: &LinuxResources, update: bool) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
         info!(
             sl!(),
             "cgroup manager set resources for container. Resources input {:?}", r
@@ -107,53 +89,49 @@ impl CgroupManager for Manager {
 
         // set cpuset and cpu reources
         if let Some(cpu) = &r.cpu {
-            set_cpu_resources(&cg, cpu)?;
+            set_cpu_resources(&self.cgroup, cpu)?;
         }
 
         // set memory resources
         if let Some(memory) = &r.memory {
-            set_memory_resources(&cg, memory, update)?;
+            set_memory_resources(&self.cgroup, memory, update)?;
         }
 
         // set pids resources
         if let Some(pids_resources) = &r.pids {
-            set_pids_resources(&cg, pids_resources)?;
+            set_pids_resources(&self.cgroup, pids_resources)?;
         }
 
         // set block_io resources
         if let Some(blkio) = &r.block_io {
-            set_block_io_resources(&cg, blkio, res)?;
+            set_block_io_resources(&self.cgroup, blkio, res)?;
         }
 
         // set hugepages resources
         if !r.hugepage_limits.is_empty() {
-            set_hugepages_resources(&cg, &r.hugepage_limits, res)?;
+            set_hugepages_resources(&self.cgroup, &r.hugepage_limits, res)?;
         }
 
         // set network resources
         if let Some(network) = &r.network {
-            set_network_resources(&cg, network, res)?;
+            set_network_resources(&self.cgroup, network, res)?;
         }
 
         // set devices resources
-        set_devices_resources(&cg, &r.devices, res)?;
+        set_devices_resources(&self.cgroup, &r.devices, res)?;
         info!(sl!(), "resources after processed {:?}", res);
 
         // apply resources
-        cg.apply(res)?;
+        self.cgroup.apply(res)?;
 
         Ok(())
     }
 
     fn get_stats(&self) -> Result<CgroupStats> {
-        let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-
         // CpuStats
-        let cpu_usage = get_cpuacct_stats(&cg);
+        let cpu_usage = get_cpuacct_stats(&self.cgroup);
 
-        let throttling_data = get_cpu_stats(&cg);
+        let throttling_data = get_cpu_stats(&self.cgroup);
 
         let cpu_stats = SingularPtrField::some(CpuStats {
             cpu_usage,
@@ -163,17 +141,17 @@ impl CgroupManager for Manager {
         });
 
         // Memorystats
-        let memory_stats = get_memory_stats(&cg);
+        let memory_stats = get_memory_stats(&self.cgroup);
 
         // PidsStats
-        let pids_stats = get_pids_stats(&cg);
+        let pids_stats = get_pids_stats(&self.cgroup);
 
         // BlkioStats
         // note that virtiofs has no blkio stats
-        let blkio_stats = get_blkio_stats(&cg);
+        let blkio_stats = get_blkio_stats(&self.cgroup);
 
         // HugetlbStats
-        let hugetlb_stats = get_hugetlb_stats(&cg);
+        let hugetlb_stats = get_hugetlb_stats(&self.cgroup);
 
         Ok(CgroupStats {
             cpu_stats,
@@ -187,10 +165,7 @@ impl CgroupManager for Manager {
     }
 
     fn freeze(&self, state: FreezerState) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
+        let freezer_controller: &FreezerController = self.cgroup.controller_of().unwrap();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let freezer_controller: &FreezerController = cg.controller_of().unwrap();
         match state {
             FreezerState::Thawed => {
                 freezer_controller.thaw()?;
@@ -207,20 +182,12 @@ impl CgroupManager for Manager {
     }
 
     fn destroy(&mut self) -> Result<()> {
-        let h = cgroups::hierarchies::auto();
+        let _ = self.cgroup.delete();
-        let h = Box::new(&*h);
-        let cg = load(h, &self.cpath);
-        if let Some(cg) = cg {
-            cg.delete();
-        }
         Ok(())
     }
 
     fn get_pids(&self) -> Result<Vec<pid_t>> {
-        let h = cgroups::hierarchies::auto();
+        let mem_controller: &MemController = self.cgroup.controller_of().unwrap();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let mem_controller: &MemController = cg.controller_of().unwrap();
         let pids = mem_controller.tasks();
         let result = pids.iter().map(|x| x.pid as i32).collect::<Vec<i32>>();
 
@@ -239,7 +206,7 @@ fn set_network_resources(
     // description can be found at https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1/net_cls.html
     let class_id = network.class_id.unwrap_or(0) as u64;
     if class_id != 0 {
-        res.network.class_id = class_id;
+        res.network.class_id = Some(class_id);
     }
 
     // set network priorities
@@ -252,7 +219,6 @@ fn set_network_resources(
         });
     }
 
-    res.network.update_values = true;
     res.network.priorities = priorities;
     Ok(())
 }
@@ -283,7 +249,6 @@ fn set_devices_resources(
         }
     }
 
-    res.devices.update_values = true;
     res.devices.devices = devices;
 
     Ok(())
@@ -295,7 +260,6 @@ fn set_hugepages_resources(
     res: &mut cgroups::Resources,
 ) -> Result<()> {
     info!(sl!(), "cgroup manager set hugepage");
-    res.hugepages.update_values = true;
     let mut limits = vec![];
 
     for l in hugepage_limits.iter() {
@@ -316,7 +280,6 @@ fn set_block_io_resources(
     res: &mut cgroups::Resources,
 ) -> Result<()> {
     info!(sl!(), "cgroup manager set block io");
-    res.blkio.update_values = true;
 
     if cg.v2() {
         res.blkio.weight = convert_blk_io_to_v2_value(blkio.weight);
@@ -607,10 +570,8 @@ lazy_static! {
 }
 
 fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
-    let cpu_controller: &CpuController = cg.controller_of().unwrap();
+    let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
-
     let stat = cpu_controller.cpu().stat;
-
     let h = lines_to_map(&stat);
 
     SingularPtrField::some(ThrottlingData {
@@ -623,39 +584,7 @@ fn get_cpu_stats(cg: &cgroups::Cgroup) -> SingularPtrField<ThrottlingData> {
 }
 
 fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
-    let cpuacct_controller: Option<&CpuAcctController> = cg.controller_of();
+    if let Some(cpuacct_controller) = cg.controller_of::<CpuAcctController>() {
-    if cpuacct_controller.is_none() {
-        if cg.v2() {
-            return SingularPtrField::some(CpuUsage {
-                total_usage: 0,
-                percpu_usage: vec![],
-                usage_in_kernelmode: 0,
-                usage_in_usermode: 0,
-                unknown_fields: UnknownFields::default(),
-                cached_size: CachedSize::default(),
-            });
-        }
-
-        // try to get from cpu controller
-        let cpu_controller: &CpuController = cg.controller_of().unwrap();
-        let stat = cpu_controller.cpu().stat;
-        let h = lines_to_map(&stat);
-        let usage_in_usermode = *h.get("user_usec").unwrap();
-        let usage_in_kernelmode = *h.get("system_usec").unwrap();
-        let total_usage = *h.get("usage_usec").unwrap();
-        let percpu_usage = vec![];
-
-        return SingularPtrField::some(CpuUsage {
-            total_usage,
-            percpu_usage,
-            usage_in_kernelmode,
-            usage_in_usermode,
-            unknown_fields: UnknownFields::default(),
-            cached_size: CachedSize::default(),
-        });
-    }
-
-    let cpuacct_controller = cpuacct_controller.unwrap();
         let cpuacct = cpuacct_controller.cpuacct();
 
         let h = lines_to_map(&cpuacct.stat);
@@ -668,6 +597,36 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let percpu_usage = line_to_vec(&cpuacct.usage_percpu);
 
+        return SingularPtrField::some(CpuUsage {
+            total_usage,
+            percpu_usage,
+            usage_in_kernelmode,
+            usage_in_usermode,
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
+        });
+    }
+
+    if cg.v2() {
+        return SingularPtrField::some(CpuUsage {
+            total_usage: 0,
+            percpu_usage: vec![],
+            usage_in_kernelmode: 0,
+            usage_in_usermode: 0,
+            unknown_fields: UnknownFields::default(),
+            cached_size: CachedSize::default(),
+        });
+    }
+
+    // try to get from cpu controller
+    let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
+    let stat = cpu_controller.cpu().stat;
+    let h = lines_to_map(&stat);
+    let usage_in_usermode = *h.get("user_usec").unwrap();
+    let usage_in_kernelmode = *h.get("system_usec").unwrap();
+    let total_usage = *h.get("usage_usec").unwrap();
+    let percpu_usage = vec![];
+
     SingularPtrField::some(CpuUsage {
         total_usage,
         percpu_usage,
@@ -679,7 +638,7 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 }
 
 fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
-    let memory_controller: &MemController = cg.controller_of().unwrap();
+    let memory_controller: &MemController = get_controller_or_return_singular_none!(cg);
 
     // cache from memory stat
     let memory = memory_controller.memory_stat();
@@ -736,7 +695,7 @@ fn get_memory_stats(cg: &cgroups::Cgroup) -> SingularPtrField<MemoryStats> {
 }
 
 fn get_pids_stats(cg: &cgroups::Cgroup) -> SingularPtrField<PidsStats> {
-    let pid_controller: &PidController = cg.controller_of().unwrap();
+    let pid_controller: &PidController = get_controller_or_return_singular_none!(cg);
 
     let current = pid_controller.get_pid_current().unwrap_or(0);
     let max = pid_controller.get_pid_max();
@@ -842,7 +801,7 @@ fn build_blkio_stats_entry(major: i16, minor: i16, op: &str, value: u64) -> Blki
 }
 
 fn get_blkio_stats_v2(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
-    let blkio_controller: &BlkIoController = cg.controller_of().unwrap();
+    let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
     let blkio = blkio_controller.blkio();
 
     let mut resp = BlkioStats::new();
@@ -870,7 +829,7 @@ fn get_blkio_stats(cg: &cgroups::Cgroup) -> SingularPtrField<BlkioStats> {
         return get_blkio_stats_v2(&cg);
     }
 
-    let blkio_controller: &BlkIoController = cg.controller_of().unwrap();
+    let blkio_controller: &BlkIoController = get_controller_or_return_singular_none!(cg);
     let blkio = blkio_controller.blkio();
 
     let mut m = BlkioStats::new();
@@ -987,9 +946,19 @@ pub fn get_mounts() -> Result<HashMap<String, String>> {
     Ok(m)
 }
 
+fn new_cgroup(
+    h: Box<dyn cgroups::Hierarchy>,
+    path: &str,
+    relative_paths: HashMap<String, String>,
+) -> Cgroup {
+    let valid_path = path.trim_start_matches('/').to_string();
+    cgroups::Cgroup::new_with_relative_paths(h, valid_path.as_str(), relative_paths)
+}
+
 impl Manager {
     pub fn new(cpath: &str) -> Result<Self> {
         let mut m = HashMap::new();
+        let mut relative_paths = HashMap::new();
 
         let paths = get_paths()?;
         let mounts = get_mounts()?;
@@ -1008,6 +977,7 @@ impl Manager {
             };
 
             m.insert(key.to_string(), p);
+            relative_paths.insert(key.to_string(), value.to_string());
         }
 
         Ok(Self {
@@ -1015,6 +985,8 @@ impl Manager {
             mounts,
             // rels: paths,
             cpath: cpath.to_string(),
+            cgroup: new_cgroup(cgroups::hierarchies::auto(), cpath, relative_paths.clone()),
+            relative_paths,
         })
     }
 
@@ -1025,18 +997,14 @@ impl Manager {
         info!(sl!(), "update_cpuset_path to: {}", guest_cpuset);
 
         let h = cgroups::hierarchies::auto();
-        let h = Box::new(&*h);
+        let root_cg = h.root_control_group();
-        let root_cg = load_or_create(h, "");
 
         let root_cpuset_controller: &CpuSetController = root_cg.controller_of().unwrap();
         let path = root_cpuset_controller.path();
         let root_path = Path::new(path);
         info!(sl!(), "root cpuset path: {:?}", &path);
 
-        let h = cgroups::hierarchies::auto();
+        let container_cpuset_controller: &CpuSetController = self.cgroup.controller_of().unwrap();
-        let h = Box::new(&*h);
-        let cg = load_or_create(h, &self.cpath);
-        let container_cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
         let path = container_cpuset_controller.path();
         let container_path = Path::new(path);
         info!(sl!(), "container cpuset path: {:?}", &path);
@@ -1056,8 +1024,6 @@ impl Manager {
                 break;
             }
             i -= 1;
-            let h = cgroups::hierarchies::auto();
-            let h = Box::new(&*h);
 
             // remove cgroup root from path
             let r_path = &paths[i]
@@ -1065,7 +1031,11 @@ impl Manager {
                 .unwrap()
                 .trim_start_matches(root_path.to_str().unwrap());
             info!(sl!(), "updating cpuset for parent path {:?}", &r_path);
-            let cg = load_or_create(h, &r_path);
+            let cg = new_cgroup(
+                cgroups::hierarchies::auto(),
+                &r_path,
+                self.relative_paths.clone(),
+            );
             let cpuset_controller: &CpuSetController = cg.controller_of().unwrap();
             cpuset_controller.set_cpus(guest_cpuset)?;
         }