diff --git a/Makefile b/Makefile index 8e856138c..e70af93e4 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ TOOLS = TOOLS += agent-ctl TOOLS += kata-ctl TOOLS += log-parser +TOOLS += log-parser-rs TOOLS += runk TOOLS += trace-forwarder diff --git a/README.md b/README.md index 6972ed278..662899ea6 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ The table below lists the remaining parts of the project: | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. | | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. | | [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. | +| [`log-parser-rs`](src/tools/log-parser-rs) | utility | Tool that aid in analyzing logs from the kata runtime. | | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. | | [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. | | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. | diff --git a/src/tools/log-parser-rs/Cargo.lock b/src/tools/log-parser-rs/Cargo.lock new file mode 100644 index 000000000..c1a19ec4e --- /dev/null +++ b/src/tools/log-parser-rs/Cargo.lock @@ -0,0 +1,899 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time 0.1.45", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "clap" +version = "4.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" +dependencies = [ + "bitflags 2.0.2", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "clap_lex" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "csv" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "cxx" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.5", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "log-parser" +version = "0.0.1" +dependencies = [ + "chrono", + "clap", + "csv", + "quick-xml", + "ron", + "serde", + "serde_json", + "serde_with", + "serde_yaml", + "slog", + "thiserror", + "toml", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "os_str_bytes" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ron" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300a51053b1cb55c80b7a9fde4120726ddf25ca241a1cbb926626f62fb136bff" +dependencies = [ + "base64", + "bitflags 1.3.2", + "serde", +] + +[[package]] +name = "rustix" +version = "0.36.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time 0.3.20", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serde_yaml" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c2d1c76a26822187a1fbb5964e3fff108bc208f02e820ab9dac1234f6b388a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.5", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +dependencies = [ + "serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" diff --git a/src/tools/log-parser-rs/Cargo.toml b/src/tools/log-parser-rs/Cargo.toml new file mode 100644 index 000000000..bcd98246b --- /dev/null +++ b/src/tools/log-parser-rs/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "log-parser" +version = "0.0.1" +edition = "2021" +authors = ["The Kata Containers community "] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" +toml = "0.4" +ron = "0.8" +quick-xml = { version = "0.28", features = ["serialize"]} +csv = "1.2" +serde_with = "2.3" +clap = { version = "4.1", features = ["derive", "cargo"] } +thiserror = "1.0" +chrono = { version = "0.4", features = ["serde"]} +slog = "2.7" diff --git a/src/tools/log-parser-rs/Makefile b/src/tools/log-parser-rs/Makefile new file mode 100644 index 000000000..5bca7cc57 --- /dev/null +++ b/src/tools/log-parser-rs/Makefile @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Gabe Venberg +# +# SPDX-License-Identifier: Apache-2.0 + +include ../../../utils.mk + +.DEFAULT_GOAL := default +default: build + +build: + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) + +static-checks-build: + @echo "INFO: static-checks-build do nothing.." + +clean: + cargo clean + +vendor: + cargo vendor + +test: + +install: + @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . + +check: standard_rust_check + +.PHONY: \ + build \ + check \ + clean \ + install \ + test \ + vendor diff --git a/src/tools/log-parser-rs/README.md b/src/tools/log-parser-rs/README.md new file mode 100644 index 000000000..a641b0030 --- /dev/null +++ b/src/tools/log-parser-rs/README.md @@ -0,0 +1,67 @@ +# `kata-log-parser` + +## Introduction + +`log-parser-rs` is a tool that combines logfiles generated by the various +system components, sorts them by time stamp, and re-displays the log entries. + +The tool is also able to check the validity of all log records, can re-format the +logs, and output them in a different format. + +For more information on the `kata-log-parser` tool, use the help command: + +``` +$ kata-log-parser --help +``` + +> **Note** this is a rewrite of the go-based `kata-log-parser` tool, and will eventually replace it. + +## Log Format + +Kata's `runtime-rs` logs are JSON objects in the following format: + +```json +{"msg":"message","level":"INFO","ts":"1970-01-01T00:00:00.000000000Z","name":"kata-runtime","version":"0.1.0","pid":"0","source":"source","subsystem":"subsystem"} +``` + +However, if `--ignore-missing-fields` is set, a log missing one or more of the following fields may be omitted: + +- `level` +- `name` +- `version` +- `pid` +- `source` +- `subsystem` + +> **Note** a log entry must be on one single line, and a line must contain only one log entry. + +## Command line opts + +The most valuable command line options are listed below: + +- `-o, --output-file ` File to output to. If not set, sends to stdout. +- `--output-format ` Sets the format of the output. Defaults to `json`, and can be set to `csv`, `json`, `ron`, `text`, `toml`, `xml`, and `yaml`. +- `-q, --quiet` Will not print invalid log entry errors to stderr. +- `-s, --strict` Any invalid log entry will halt the program. + +For a comprehensive (and guaranteed up to date) list, please run `log-parser-rs --help`. + +## Usage + +1. Make sure containerd is in [debug mode](https://github.com/kata-containers/documentation/blob/master/Developer-Guide.md#enabling-full-containerd-debug) +1. Make sure you are running runtime-rs: + ``` + $ containerd-shim-kata-v2 --version|grep -qi rust && echo rust || echo golang + ``` +1. Collect the logs (alternatively to journal clearing you may consider constraining collected logs by adding `--since=`). + ``` + $ sudo journalctl -q -o cat -a -t kata | grep "^{" > ./kata.log ./kata.log + ``` +1. Ensure the logs are readable: + ``` + $ sudo chown $USER *.log + ``` +1. Process the logs: + ``` + $ log-parser-rs kata.log -o out.log + ``` diff --git a/src/tools/log-parser-rs/src/args.rs b/src/tools/log-parser-rs/src/args.rs new file mode 100644 index 000000000..1980fcc35 --- /dev/null +++ b/src/tools/log-parser-rs/src/args.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use clap::{Parser, ValueEnum}; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[command(name="kata-log-parser", author, version, about, long_about = None)] // Read from `Cargo.toml` +pub struct Cli { + pub input_file: Vec, + + #[arg(short, long)] + pub output_file: Option, + + #[arg(short, long, help = "check log files and only display output on error")] + pub check_only: bool, + + #[arg(long, help = "error if any files are empty")] + pub error_if_file_empty: bool, + + #[arg(long, help = "error if all logfiles are empty")] + pub error_if_no_records: bool, + + #[arg( + long, + help = "do not make an error for lines with no pid, source, name, or level" + )] + pub ignore_missing_fields: bool, + + #[arg( + short, + long, + help = "suppress warning messages that would otherwise go to stderr." + )] + pub quiet: bool, + + #[arg( + short, + long, + help = "do not tolerate misformed agent messages (may be caused by non-Kata Containers log lines)" + )] + pub strict: bool, + + #[arg(long, value_enum, default_value_t = OutputFormat::Json, help="set the output format")] + pub output_format: OutputFormat, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum OutputFormat { + Csv, + Json, + Ron, + Text, + Toml, + Xml, + Yaml, +} diff --git a/src/tools/log-parser-rs/src/log_message.rs b/src/tools/log-parser-rs/src/log_message.rs new file mode 100644 index 000000000..9caf849c6 --- /dev/null +++ b/src/tools/log-parser-rs/src/log_message.rs @@ -0,0 +1,467 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{fmt::Debug, fmt::Display, str::FromStr}; + +use chrono::{DateTime, Utc}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_with::{ + serde_as, skip_serializing_none, DeserializeFromStr, DisplayFromStr, SerializeDisplay, +}; +use thiserror::Error; + +pub trait AnyLogMessage: Serialize + DeserializeOwned + Debug { + fn get_timestamp(&self) -> DateTime; +} + +#[serde_as] +#[skip_serializing_none] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)] +pub struct LogMessage { + pub level: Option, + + #[serde(rename = "msg")] + pub message: String, + + pub name: Option, + + #[serde_as(as = "Option")] + pub pid: Option, + + pub source: Option, + + pub subsystem: Option, + + #[serde_as(as = "DisplayFromStr")] + #[serde(rename = "ts")] + pub timestamp: DateTime, +} + +impl AnyLogMessage for LogMessage { + fn get_timestamp(&self) -> DateTime { + self.timestamp + } +} + +//totally abusing serde to easily display this. +impl Display for LogMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)? + ) + } +} + +#[serde_as] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default)] +pub struct StrictLogMessage { + pub level: LogLevel, + + #[serde(rename = "msg")] + pub message: String, + + pub name: String, + + #[serde_as(as = "DisplayFromStr")] + pub pid: usize, + + pub source: String, + + pub subsystem: String, + + #[serde_as(as = "DisplayFromStr")] + #[serde(rename = "ts")] + pub timestamp: DateTime, +} + +impl AnyLogMessage for StrictLogMessage { + fn get_timestamp(&self) -> DateTime { + self.timestamp + } +} + +//totally abusing serde to easily display this. +impl Display for StrictLogMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_string_pretty(&self).map_err(|_| std::fmt::Error)? + ) + } +} + +// A newtype for slog::Level, as it does not implement Serialize and Deserialize. +#[derive(Debug, SerializeDisplay, DeserializeFromStr, PartialEq, Eq)] +pub struct LogLevel(slog::Level); + +#[derive(Debug, Error)] +pub enum LevelError { + #[error("invalid slog level: {0}")] + InvalidLevel(String), +} + +impl Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}", self.0.as_str())) + } +} + +impl Default for LogLevel { + fn default() -> Self { + LogLevel(slog::Level::Info) + } +} + +impl FromStr for LogLevel { + type Err = LevelError; + fn from_str(s: &str) -> Result { + let level = match s.to_lowercase().as_str() { + //need to accept both the short and long string versions. + "critical" | "crit" => slog::Level::Critical, + "error" | "erro" => slog::Level::Error, + "warning" | "warn" => slog::Level::Warning, + "info" => slog::Level::Info, + "debug" | "debg" => slog::Level::Debug, + "trace" | "trce" => slog::Level::Trace, + _ => return Err(LevelError::InvalidLevel(s.to_string())), + }; + + Ok(LogLevel(level)) + } +} + +//TODO: add tests for serialization. +#[cfg(test)] +mod test { + use super::*; + use crate::log_parser_error::LogParserError; + + #[test] + fn parse_string() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: Some("kata-runtime".to_string()), + pid: Some(3327263), + source: Some("foo".to_string()), + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + fn parse_string_with_missing_fields() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: None, + source: Some("foo".to_string()), + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + #[should_panic] + fn parse_error() { + let log = "random non-kata log message"; + serde_json::from_str::(log).unwrap(); + } + + #[test] + fn parse_string_strict() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#; + let result = Ok(StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }); + assert_eq!( + serde_json::from_str(log) + .map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + result + ) + } + + #[test] + #[should_panic] + fn parse_string_with_missing_fields_strict() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","version":"0.1.0","subsystem":"hypervisor","source":"foo"}"#; + println!( + "{:?}", + serde_json::from_str::(log).unwrap() + ); + } + + #[test] + #[should_panic] + fn parse_error_strict() { + let log = "random non-kata log message"; + serde_json::from_str::(log).unwrap(); + } + + #[test] + fn serialize_json_strict() { + let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","name":"kata-runtime","pid":"3327263","source":"foo","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_json::to_string(&log).unwrap()); + } + + #[test] + fn serialize_json() { + let result = r#"{"level":"DEBUG","msg":"vmm-master thread is uninitialized or has exited.","pid":"3327263","subsystem":"hypervisor","ts":"2023-03-15 14:17:02.526992506 UTC"}"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_json::to_string(&log).unwrap()); + } + + #[test] + fn serialize_csv_strict() { + let result = r#"level,msg,name,pid,source,subsystem,ts +DEBUG,vmm-master thread is uninitialized or has exited.,kata-runtime,3327263,foo,hypervisor,2023-03-15 14:17:02.526992506 UTC +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + let mut csv_writer = csv::Writer::from_writer(vec![]); + csv_writer.serialize(&log).unwrap(); + let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap(); + assert_eq!(result, output); + } + + #[test] + fn serialize_csv() { + let result = r#"level,msg,pid,subsystem,ts +DEBUG,vmm-master thread is uninitialized or has exited.,3327263,hypervisor,2023-03-15 14:17:02.526992506 UTC +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + let mut csv_writer = csv::Writer::from_writer(vec![]); + csv_writer.serialize(&log).unwrap(); + let output = String::from_utf8(csv_writer.into_inner().unwrap()).unwrap(); + assert_eq!(result, output); + } + + #[test] + fn serialize_ron_strict() { + let result = r#"(level:"DEBUG",msg:"vmm-master thread is uninitialized or has exited.",name:"kata-runtime",pid:"3327263",source:"foo",subsystem:"hypervisor",ts:"2023-03-15 14:17:02.526992506 UTC")"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, ron::to_string(&log).unwrap()); + } + + #[test] + fn serialize_ron() { + let result = r#"(level:Some("DEBUG"),msg:"vmm-master thread is uninitialized or has exited.",pid:Some("3327263"),subsystem:Some("hypervisor"),ts:"2023-03-15 14:17:02.526992506 UTC")"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, ron::to_string(&log).unwrap()); + } + + #[test] + fn serialize_toml_strict() { + let result = r#"level = "DEBUG" +msg = "vmm-master thread is uninitialized or has exited." +name = "kata-runtime" +pid = "3327263" +source = "foo" +subsystem = "hypervisor" +ts = "2023-03-15 14:17:02.526992506 UTC" +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, toml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_toml() { + let result = r#"level = "DEBUG" +msg = "vmm-master thread is uninitialized or has exited." +pid = "3327263" +subsystem = "hypervisor" +ts = "2023-03-15 14:17:02.526992506 UTC" +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, toml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_xml_strict() { + let result = r#"DEBUGvmm-master thread is uninitialized or has exited.kata-runtime3327263foohypervisor2023-03-15 14:17:02.526992506 UTC"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, quick_xml::se::to_string(&log).unwrap()); + } + + #[test] + fn serialize_xml() { + let result = r#"DEBUGvmm-master thread is uninitialized or has exited.3327263hypervisor2023-03-15 14:17:02.526992506 UTC"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, quick_xml::se::to_string(&log).unwrap()); + } + + #[test] + fn serialize_yaml_strict() { + let result = r#"level: DEBUG +msg: vmm-master thread is uninitialized or has exited. +name: kata-runtime +pid: '3327263' +source: foo +subsystem: hypervisor +ts: 2023-03-15 14:17:02.526992506 UTC +"#; + let log = StrictLogMessage { + level: LogLevel(slog::Level::Debug), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: "kata-runtime".to_string(), + pid: 3327263, + source: "foo".to_string(), + subsystem: "hypervisor".to_string(), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_yaml::to_string(&log).unwrap()); + } + + #[test] + fn serialize_yaml() { + let result = r#"level: DEBUG +msg: vmm-master thread is uninitialized or has exited. +pid: '3327263' +subsystem: hypervisor +ts: 2023-03-15 14:17:02.526992506 UTC +"#; + let log = LogMessage { + level: Some(LogLevel(slog::Level::Debug)), + message: "vmm-master thread is uninitialized or has exited.".to_string(), + name: None, + pid: Some(3327263), + source: None, + subsystem: Some("hypervisor".to_string()), + timestamp: chrono::DateTime::parse_from_rfc3339("2023-03-15T14:17:02.526992506Z") + .unwrap() + .into(), + }; + assert_eq!(result, serde_yaml::to_string(&log).unwrap()); + } +} diff --git a/src/tools/log-parser-rs/src/log_parser_error.rs b/src/tools/log-parser-rs/src/log_parser_error.rs new file mode 100644 index 000000000..4af197d27 --- /dev/null +++ b/src/tools/log-parser-rs/src/log_parser_error.rs @@ -0,0 +1,58 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{error::Error, path::PathBuf}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum LogParserError { + #[error("Unknown Error")] + Unknown(Box), + + #[error("Input file '{0}' cannot be found")] + InputFileNotFound(PathBuf), + + #[error("Input file '{0}' does not contain any valid logs")] + FileEmpty(PathBuf), + + #[error("No permission to open '{0}'")] + InputFilePermissionError(PathBuf), + + #[error("No permission to write to '{0}'")] + OutputFilePermissionError(PathBuf), + + #[error("Log parsing error: {0} with string {1}")] + ParsingError(serde_json::Error, String), + + #[error("Error serializing {0}: {1}")] + SerializationError(String, Box), + + #[error("No logs in any file")] + NoRecordsError(), +} + +impl PartialEq for LogParserError { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Unknown(l0), Self::Unknown(r0)) => l0.to_string() == r0.to_string(), + (Self::InputFileNotFound(l0), Self::InputFileNotFound(r0)) => l0 == r0, + (Self::FileEmpty(l0), Self::FileEmpty(r0)) => l0 == r0, + (Self::InputFilePermissionError(l0), Self::InputFilePermissionError(r0)) => l0 == r0, + (Self::OutputFilePermissionError(l0), Self::OutputFilePermissionError(r0)) => l0 == r0, + //serde_json::Error does not impl partialeq, but for testing cases a quick and dirty + //string comparison works well enough. + (Self::ParsingError(l0, l1), Self::ParsingError(r0, r1)) => { + l0.to_string() == r0.to_string() && l1 == r1 + } + //this catch all returns whether the two enums are the same variant type. eg, + //core::mem:discriminant(LogParserError::Unkown)==core:mem:discriminant(LogParserError::Unkown) + //is true, but it would not be true, for say, Unkown and InputFilePermissionError. + //Note that it only compares the variants, not the contents of the variants, hence the + //need for the above branches. + _ => core::mem::discriminant(self) == core::mem::discriminant(other), + } + } +} + +impl Eq for LogParserError {} diff --git a/src/tools/log-parser-rs/src/main.rs b/src/tools/log-parser-rs/src/main.rs new file mode 100644 index 000000000..72c406a59 --- /dev/null +++ b/src/tools/log-parser-rs/src/main.rs @@ -0,0 +1,68 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +#![warn(unused_crate_dependencies)] +#![warn(missing_debug_implementations)] + +mod args; +mod log_message; +mod log_parser_error; +mod output_file; +mod parse_file; +mod process_logs; + +use crate::args::Cli; +use crate::log_message::StrictLogMessage; +use crate::log_parser_error::LogParserError; +use crate::output_file::*; +use crate::parse_file::*; +use crate::process_logs::*; +use clap::Parser; +use log_message::AnyLogMessage; +use log_message::LogMessage; +use std::process::exit; + +fn handle_logs(cli: Cli) -> Result<(), LogParserError> { + let mut logs = Vec::new(); + + for file in &cli.input_file { + let in_file = open_file_into_memory(file)?; + let file_logs = filter_errors(parse_log::(in_file), &cli)?; + + if cli.error_if_file_empty && file_logs.is_empty() { + return Err(LogParserError::FileEmpty(file.to_path_buf())); + } + + logs.extend(file_logs) + } + + if cli.error_if_no_records && logs.is_empty() { + return Err(LogParserError::NoRecordsError()); + } + if cli.check_only { + return Ok(()); + } + + sort_logs(&mut logs); + output_file(logs, &cli)?; + Ok(()) +} + +//needed another layer of function call in order to genericize over both LogMessage and +//StrictLogMessage. +fn real_main() -> std::result::Result<(), LogParserError> { + let cli = Cli::parse(); + if cli.ignore_missing_fields { + handle_logs::(cli) + } else { + handle_logs::(cli) + } +} + +fn main() { + if let Err(e) = real_main() { + eprintln!("ERROR: {:#}", e); + exit(1); + } +} diff --git a/src/tools/log-parser-rs/src/output_file.rs b/src/tools/log-parser-rs/src/output_file.rs new file mode 100644 index 000000000..eb6ddf146 --- /dev/null +++ b/src/tools/log-parser-rs/src/output_file.rs @@ -0,0 +1,149 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs::File, + io::{ErrorKind, Write}, + path::Path, +}; + +use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError}; + +/// a simple dispatcher method that outputs the deserialized result of the parsed logs according to +/// the CLI arguments. +/// +/// # Errors +/// +/// If outputting to a file, may return a LogParserError having to do with opening and writing to +/// the output file. +pub(crate) fn output_file( + contents: Vec, + options: &Cli, +) -> Result<(), LogParserError> { + let serializer = choose_formatting(options); + if let Some(out_file) = &options.output_file { + write_logs_to_file(out_file, contents, serializer)?; + } else { + print_logs(contents, serializer)?; + }; + Ok(()) +} + +fn choose_formatting( + options: &Cli, +) -> fn(Vec) -> Result { + match options.output_format { + crate::args::OutputFormat::Csv => serialize_csv, + crate::args::OutputFormat::Json => serialize_json, + crate::args::OutputFormat::Ron => serialize_ron, + crate::args::OutputFormat::Text => serialize_text, + crate::args::OutputFormat::Toml => serialize_toml, + crate::args::OutputFormat::Xml => serialize_xml, + crate::args::OutputFormat::Yaml => serialize_yaml, + } +} + +fn serialize_text(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| Ok(format!("{:?}", r))) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_json(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + serde_json::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_toml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + toml::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_yaml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + serde_yaml::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_xml(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + quick_xml::se::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_ron(input: Vec) -> Result { + Ok(input + .iter() + .map(|r| { + ron::to_string(r) + .map_err(|e| LogParserError::SerializationError(format!("{:?}", r), Box::new(e))) + }) + .collect::, LogParserError>>()? + .join("\n")) +} + +fn serialize_csv(input: Vec) -> Result { + let mut csv_writer = csv::Writer::from_writer(vec![]); + for record in input { + csv_writer.serialize(&record).map_err(|e| { + LogParserError::SerializationError(format!("{:?}", record), Box::new(e)) + })?; + } + String::from_utf8( + csv_writer + .into_inner() + .map_err(|e| LogParserError::Unknown(Box::new(e)))?, + ) + .map_err(|e| LogParserError::Unknown(Box::new(e))) +} + +fn write_logs_to_file( + out_file: &Path, + contents: Vec, + serializer: fn(Vec) -> Result, +) -> Result<(), LogParserError> { + let mut file = File::create(out_file).map_err(|e| map_out_io_err(e, out_file))?; + file.write_all(serializer(contents)?.as_bytes()) + .map_err(|e| LogParserError::Unknown(Box::new(e)))?; + Ok(()) +} + +fn print_logs( + contents: Vec, + serializer: fn(Vec) -> Result, +) -> Result<(), LogParserError> { + print!("{}", serializer(contents)?); + Ok(()) +} + +fn map_out_io_err(e: std::io::Error, out_file: &Path) -> LogParserError { + match e.kind() { + ErrorKind::PermissionDenied => LogParserError::OutputFilePermissionError(out_file.into()), + _ => LogParserError::Unknown(Box::new(e)), + } +} diff --git a/src/tools/log-parser-rs/src/parse_file.rs b/src/tools/log-parser-rs/src/parse_file.rs new file mode 100644 index 000000000..3304aebe2 --- /dev/null +++ b/src/tools/log-parser-rs/src/parse_file.rs @@ -0,0 +1,85 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::{log_message::AnyLogMessage, log_parser_error::LogParserError}; +use std::{fs, io::ErrorKind, path::Path}; + +/// Reads the entire file into memory as a string. +/// +/// # Errors +/// +/// This function will return an error if opening the file returns an error. +/// FileNotFound and FilePermissionError *should* be the only error types that can happen, and they +/// are wrapped in the appropriate LogParserError types. Any other error will panic. +/// +/// # Panic +/// +/// Will panic if File::open returns an error other than NotFound or PermissionDenied. +/// This *should* not happen. +/// +/// # Gochas +/// +/// Memory use can be unexpectedly high, as entire file is read into memory. +pub(crate) fn open_file_into_memory(inputfile: &Path) -> Result { + match fs::read_to_string(inputfile) { + Ok(s) => Ok(s), + Err(e) => match e.kind() { + ErrorKind::NotFound => Err(LogParserError::InputFileNotFound(inputfile.to_path_buf())), + ErrorKind::PermissionDenied => Err(LogParserError::InputFilePermissionError( + inputfile.to_path_buf(), + )), + _ => Err(LogParserError::Unknown(Box::new(e))), + }, + } +} + +/// Parses a series of logs from a string, returning an array of log messages and parsing errors. +/// +/// # Errors +/// +/// Will pass any errors from serde in to the output array +pub(crate) fn parse_log(in_file: String) -> Vec> +where + O: AnyLogMessage, +{ + let mut output = Vec::new(); + for line in in_file.lines() { + let logentry = serde_json::from_str::(line) + .map_err(|e| LogParserError::ParsingError(e, line.to_string())); + output.push(logentry); + } + output +} + +#[cfg(test)] +mod test { + use super::*; + use crate::log_message::LogMessage; + + #[test] + fn parse_strings() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"} +{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#; + let result = vec![ + serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + serde_json::from_str(r#"{"msg":"resource clean up","level":"INFO","ts":"2023-03-15T14:17:02.527047136Z","subsystem":"virt-container","name":"kata-runtime","pid":"3327263","version":"0.1.0","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + ]; + assert_eq!(parse_log::(log.into()), result) + } + #[test] + fn parse_mixed() { + let log = r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"} +Random Kernel Message"#; + let result = vec![ + serde_json::from_str(r#"{"msg":"vmm-master thread is uninitialized or has exited.","level":"DEBG","ts":"2023-03-15T14:17:02.526992506Z","pid":"3327263","version":"0.1.0","name":"kata-runtime","subsystem":"hypervisor","source":"foo"}"#).map_err(|e| LogParserError::ParsingError(e, r#"Will not happen"#.to_string())), + + Err(LogParserError::ParsingError( + serde_json::from_str::("Random Kernel Message") + .err() + .unwrap() + , "Random Kernel Message".to_string())), + ]; + assert_eq!(parse_log::(log.into()), result) + } +} diff --git a/src/tools/log-parser-rs/src/process_logs.rs b/src/tools/log-parser-rs/src/process_logs.rs new file mode 100644 index 000000000..966124fa0 --- /dev/null +++ b/src/tools/log-parser-rs/src/process_logs.rs @@ -0,0 +1,90 @@ +// Copyright (c) 2023 Gabe Venberg +// +// SPDX-License-Identifier: Apache-2.0 + +use crate::{args::Cli, log_message::AnyLogMessage, log_parser_error::LogParserError}; + +/// Calls functions to either check for errors, or to filter them out, discarding them or printing +/// them to stderr. +/// +/// # Errors +/// +/// If cli.strict is true, will return the first error it finds in input (if any) +pub(crate) fn filter_errors( + input: Vec>, + cli: &Cli, +) -> Result, LogParserError> { + if cli.strict { + find_errors(input) + } else if cli.quiet { + Ok(filter_errors_quiet(input)) + } else { + Ok(filter_errors_to_stderr(input)) + } +} + +/// checks if any errors are in the passed log vector. If there are none, returns a vec of O +/// (stripping the containing Result). If there are some, returns the first found error. +fn find_errors( + input: Vec>, +) -> Result, LogParserError> { + // yes, this is only the one line, but the behavior of collect() here is non-obvious. In short, + // as Result implements IntoIter, you can go from a `Vec` to a `Result, E>` + // with just collect() + input.into_iter().collect() +} + +/// removes all LogParserErrors, returning a vec of just O +fn filter_errors_quiet(input: Vec>) -> Vec { + input.into_iter().filter_map(|l| l.ok()).collect() +} + +/// removes all LogParserErrors, returning a vec of just O, and prints any errors to stderr. +fn filter_errors_to_stderr(input: Vec>) -> Vec { + input + .into_iter() + .filter_map(|l| match l { + Ok(log) => Some(log), + Err(e) => { + eprintln!("{}", e); + None + } + }) + .collect() +} + +/// does what it says on the tin. Sorts a vec of logs in place by their timestamp. +pub(crate) fn sort_logs(input: &mut [O]) { + input.sort_by_key(|l| l.get_timestamp()); +} + +#[cfg(test)] +mod test { + use super::*; + use crate::log_message::LogMessage; + #[test] + fn error_filter() { + let unclean_logs = vec![ + Ok(LogMessage::default()), + Err(LogParserError::SerializationError( + "test error".to_string(), + Box::new(slog::Error::Fmt(std::fmt::Error)), + )), + ]; + let logs = vec![LogMessage::default()]; + assert_eq!(filter_errors_quiet(unclean_logs), logs) + } + + #[test] + fn error_filter_to_stderr() { + let unclean_logs = vec![ + Ok(LogMessage::default()), + Err(LogParserError::SerializationError( + "test error".to_string(), + Box::new(slog::Error::Fmt(std::fmt::Error)), + )), + ]; + let logs = vec![LogMessage::default()]; + assert_eq!(filter_errors_to_stderr(unclean_logs), logs) + } +}