tools: remove libtelemetry code from crashlog

Remove the code that deals with the libtelemetry API and infrastructure.
Libtelemetry is used in Clear Linux only and not available (out of the
box) on Ubuntu or Yocto (which we use in our reference stack).

Tracked-On: #5653
Signed-off-by: Geoffroy Van Cutsem <geoffroy.vancutsem@intel.com>
This commit is contained in:
Geoffroy Van Cutsem 2021-01-21 10:47:35 +01:00 committed by wenlingz
parent 0de004e5c9
commit baea9ecb44
6 changed files with 5 additions and 576 deletions

View File

@ -71,13 +71,6 @@ ifeq ($(strip $(LIB_EXIST)),lsystemd)
EXTRA_LIBS := -lsystemd
endif
PKG_CONFIG := $(shell export PKG_CONFIG_PATH=$(PKG_CONFIG_PATH); \
pkg-config --libs libtelemetry)
LIB_EXIST := $(findstring ltelemetry, $(PKG_CONFIG))
ifeq ($(strip $(LIB_EXIST)),ltelemetry)
CFLAGS += -DHAVE_TELEMETRICS_CLIENT
EXTRA_LIBS += -ltelemetry
endif
export CFLAGS
export LDFLAGS
export EXTRA_LIBS
@ -109,13 +102,9 @@ install:
@install -d $(DESTDIR)$(datadir)/acrn/crashlog
@install -p -D -m 0644 data/40-watchdog.conf $(DESTDIR)$(datadir)/acrn/crashlog
@install -p -D -m 0644 data/80-coredump.conf $(DESTDIR)$(datadir)/acrn/crashlog
@install -d $(DESTDIR)$(datadir)/defaults/telemetrics/
@install -p -D -m 0644 data/acrnprobe.xml $(DESTDIR)$(datadir)/defaults/telemetrics/
@install -d $(DESTDIR)$(systemd_unitdir)/system/
@install -p -D -m 0644 data/acrnprobe.service $(DESTDIR)$(systemd_unitdir)/system/
@install -p -D -m 0644 data/usercrash.service $(DESTDIR)$(systemd_unitdir)/system/
@install -d $(DESTDIR)$(libdir)/tmpfiles.d
@install -p -D -m 0644 data/acrn-crashlog-dirs.conf $(DESTDIR)$(libdir)/tmpfiles.d/
.PHONY:uninstall
uninstall:
@ -144,15 +133,9 @@ uninstall:
@if [ -e "$(DESTDIR)$(datadir)/acrn/crashlog/80-coredump.conf" ];then \
$(RM) $(DESTDIR)$(datadir)/acrn/crashlog/80-coredump.conf; \
fi
@if [ -e "$(DESTDIR)$(datadir)/defaults/telemetrics/acrnprobe.xml" ];then \
$(RM) $(DESTDIR)$(datadir)/defaults/telemetrics/acrnprobe.xml; \
fi
@if [ -e "$(DESTDIR)$(systemd_unitdir)/system/acrnprobe.service" ];then \
$(RM) $(DESTDIR)$(systemd_unitdir)/system/acrnprobe.service; \
fi
@if [ -e "$(DESTDIR)$(systemd_unitdir)/system/usercrash.service" ];then \
$(RM) $(DESTDIR)$(systemd_unitdir)/system/usercrash.service; \
fi
@if [ -e "$(DESTDIR)$(libdir)/tmpfiles.d/acrn-crashlog-dirs.conf" ];then \
$(RM) $(DESTDIR)$(libdir)/tmpfiles.d/acrn-crashlog-dirs.conf; \
fi

View File

@ -25,7 +25,6 @@ The ``ACRN-Crashlog`` tool depends on the following libraries
- OpenSSL
- libxml2
- systemd
- telemetrics-client-dev (optional, detected at build time)
- libblkid
- e2fsprogs
@ -75,22 +74,10 @@ Then it will show:
.. code-block:: console
... Backup core pattern to /var/log/crashlog/default_core_pattern
'/usr/share/defaults/telemetrics/telemetrics.conf' ->
'/etc/telemetrics/telemetrics.conf'
... Set server_delivery_enabled=false in /etc/telemetrics/telemetrics.conf
... Set record_retention_enabled=true in /etc/telemetrics/telemetrics.conf
'/usr/share/acrn/crashlog/40-watchdog.conf' ->
'/etc/systemd/system.conf.d/40-watchdog.conf'
'/usr/share/acrn/crashlog/80-coredump.conf' ->
'/etc/sysctl.d/80-coredump.conf'
Created symlink /etc/systemd/system/hprobe.timer -> /dev/null.
Created symlink /etc/systemd/system/telemd-update-trigger.service -> /dev/null.
Created symlink /etc/systemd/system/pstore-clean.service -> /dev/null.
Created symlink /etc/systemd/system/pstore-probe.service -> /dev/null.
Created symlink /etc/systemd/system/oops-probe.service -> /dev/null.
Created symlink /etc/systemd/system/klogscanner.service -> /dev/null.
Created symlink /etc/systemd/system/journal-probe.service -> /dev/null.
Created symlink /etc/systemd/system/bert-probe.service -> /dev/null.
Created symlink /etc/systemd/system/multi-user.target.wants/acrnprobe.service -> /usr/lib/systemd/system/acrnprobe.service.
Created symlink /etc/systemd/system/multi-user.target.wants/usercrash.service -> /usr/lib/systemd/system/usercrash.service.
*** Please reboot your system. ***
@ -113,18 +100,8 @@ Then it will show:
Removed /etc/systemd/system/multi-user.target.wants/acrnprobe.service.
Removed /etc/systemd/system/multi-user.target.wants/usercrash.service.
Removed /etc/systemd/system/hprobe.timer.
Removed /etc/systemd/system/telemd-update-trigger.service.
Removed /etc/systemd/system/pstore-clean.service.
Removed /etc/systemd/system/pstore-probe.service.
Removed /etc/systemd/system/oops-probe.service.
Removed /etc/systemd/system/klogscanner.service.
Removed /etc/systemd/system/journal-probe.service.
Removed /etc/systemd/system/bert-probe.service.
removed '/etc/sysctl.d/80-coredump.conf'
removed '/etc/systemd/system.conf.d/40-watchdog.conf'
... Set server_delivery_enabled=true in /etc/telemetrics/telemetrics.conf
... Set record_retention_enabled=false in /etc/telemetrics/telemetrics.conf
*** Please reboot your system. ***
Follow the hints to reboot the system:
@ -143,47 +120,17 @@ It will show the status of the related services like:
.. code-block:: console
telemprobd : active
telempostd : active
acrnprobe : inactive
usercrash : inactive
Usage
*****
The ``acrnprobe`` tool can work in two ways according to the existence of
telemetrics-client on the system:
The ``acrnprobe`` tool provides ``history_event`` (under
``/var/log/crashlog/history_event``) to record ACRN-related events and
crash information.
1. If telemetrics-client doesn't exist on the system, ``acrnprobe`` provides
``history_event`` (under ``/var/log/crashlog/history_event``) to manage the
crash and events records on the platform. But in this case, the records
can't be delivered to the backend.
2. If telemetrics-client exists on the system, ``acrnprobe`` works as a probe
of the telemetrics-client: it runs as a daemon autostarted when the system
boots, and sends the crashlog path to the telemetrics-client that records
events of interest and reports them to the backend using ``telemd`` the
telemetrics daemon. The workflow of ``acrnprobe`` and
telemetrics-client is shown in :numref:`crashlog-workflow`:
.. graphviz:: images/crashlog-workflow.dot
:name: crashlog-workflow
:align: center
:caption: acrnprobe and telemetrics-client workflow
Crashlog can be retrieved with ``telem_journal`` command:
.. code-block:: none
$ telem_journal -i
.. note::
For more details of telemetrics, please refer the `telemetrics-client`_ and
`telemetrics-backend`_ website.
``ACRN-Crashlog`` also provides a tool ``debugger`` to dump the specific
``ACRN-Crashlog`` also provides a tool called ``debugger`` to dump specific
process information:
.. code-block:: none
@ -241,6 +188,3 @@ userspace. It works in Client/Server model. Server is autostarted, and client is
configured in ``core_pattern`` or ``coredump-wrapper``, which will be
triggered once crash occurs in userspace.
For more detail on ``usercrash``, please refer :ref:`usercrash_doc`.
.. _`telemetrics-client`: https://github.com/clearlinux/telemetrics-client
.. _`telemetrics-backend`: https://github.com/clearlinux/telemetrics-backend

View File

@ -25,14 +25,6 @@
#include "log_sys.h"
#include "loop.h"
#ifdef HAVE_TELEMETRICS_CLIENT
#include "telemetry.h"
#define CRASH_SEVERITY 4
#define INFO_SEVERITY 2
#endif
static int crashlog_check_space(void)
{
struct sender_t *crashlog = get_sender_by_name("crashlog");
@ -216,93 +208,6 @@ static void get_log_by_type(const char *despath, const struct log_t *log,
if (log->deletesource && !strcmp("true", log->deletesource))
remove(srcpath);
}
#ifdef HAVE_TELEMETRICS_CLIENT
static int telemd_send_data(char *payload, char *eventid, uint32_t severity,
char *class)
{
int res;
struct telem_ref *handle = NULL;
const uint32_t version = 1;
res = tm_create_record(&handle, severity, class, version);
if (res < 0) {
LOGE("failed to create record: %s\n",
strerror(-res));
goto fail;
}
/* eventid with 32 character length */
if (eventid) {
res = tm_set_event_id(handle, eventid);
if (res < 0) {
LOGE("failed to set eventid: %s\n", strerror(-res));
goto free;
}
}
res = tm_set_payload(handle, payload);
if (res < 0) {
LOGE("failed to set payload: %s\n", strerror(-res));
goto free;
}
res = tm_send_record(handle);
if (res < 0) {
LOGE("failed to send record: %s\n", strerror(-res));
goto free;
}
tm_free_record(handle);
return 0;
free:
tm_free_record(handle);
fail:
return -1;
}
static void telemd_get_log(const char *dir, char *eventid,
uint32_t severity, char *class)
{
char *files[512];
char *msg;
int count;
int i;
if (!dir)
goto send_nologs;
/* send logs */
count = lsdir(dir, files, ARRAY_SIZE(files));
if (count < 0) {
LOGE("lsdir (%s) failed, error (%s)\n", dir, strerror(-count));
return;
}
if (count > 2) {
for (i = 0; i < count; i++) {
if (strstr(files[i], "/.") || strstr(files[i], "/.."))
continue;
telemd_send_data(files[i], eventid, severity, class);
}
while (count > 0)
free(files[--count]);
return;
} else if (count == 2) {
while (count > 0)
free(files[--count]);
}
send_nologs:
if (asprintf(&msg, "no logs provided, check probe's log.") == -1) {
LOGE("failed to generate msg, out of memory\n");
return;
}
telemd_send_data(msg, eventid, severity, class);
free(msg);
}
#endif
static void crashlog_get_log(struct log_t *log, void *data)
{
@ -369,325 +274,6 @@ static void crashlog_get_log(struct log_t *log, void *data)
LOGW("get (%s) spend %ds\n", log->name, spent);
}
#ifdef HAVE_TELEMETRICS_CLIENT
static void telemd_send_crash(struct event_t *e, char *eventid)
{
struct crash_t *crash;
char *class;
int ret;
crash = (struct crash_t *)e->private;
ret = asprintf(&class, "clearlinux/crash/%s", crash->name);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
return;
}
telemd_get_log(e->dir, eventid, CRASH_SEVERITY, class);
free(class);
}
static void telemd_send_info(struct event_t *e, char *eventid)
{
struct info_t *info;
char *class;
int ret;
info = (struct info_t *)e->private;
ret = asprintf(&class, "clearlinux/info/%s", info->name);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
return;
}
telemd_get_log(e->dir, eventid, INFO_SEVERITY, class);
free(class);
}
static void telemd_send_uptime(void)
{
struct sender_t *telemd = get_sender_by_name("telemd");
struct uptime_t *uptime;
char *class;
char boot_time[UPTIME_SIZE];
int hours;
int ret;
static int uptime_hours;
static int loop_uptime_event = 1;
if (!telemd)
return;
ret = get_uptime_string(boot_time, &hours);
if (ret < 0) {
LOGE("cannot get uptime - %s\n", strerror(-ret));
return;
}
uptime = telemd->uptime;
if (cfg_atoi(uptime->eventhours, uptime->eventhours_len,
&uptime_hours) == -1)
return;
if (hours / uptime_hours >= loop_uptime_event) {
char *content;
loop_uptime_event = (hours / uptime_hours) + 1;
ret = asprintf(&class, "clearlinux/uptime/%s", boot_time);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
return;
}
ret = asprintf(&content, "system boot time: %s", boot_time);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
free(class);
return;
}
telemd_send_data(content, NULL, INFO_SEVERITY, class);
free(class);
free(content);
}
}
static void telemd_send_reboot(char *eventid)
{
struct sender_t *telemd = get_sender_by_name("telemd");
char *class;
char reason[REBOOT_REASON_SIZE];
int ret;
if (!telemd)
return;
if (swupdated(telemd)) {
char *content;
ret = asprintf(&class, "clearlinux/swupdate/-");
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
return;
}
ret = asprintf(&content, "system update to: %s",
gbuildversion);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
free(class);
return;
}
telemd_send_data(content, NULL, INFO_SEVERITY, class);
free(class);
free(content);
}
read_startupreason(reason, sizeof(reason));
ret = asprintf(&class, "clearlinux/reboot/%s", reason);
if (ret < 0) {
LOGE("compute string failed, out of memory\n");
return;
}
telemd_send_data("reboot", eventid, INFO_SEVERITY, class);
free(class);
}
static void telemd_send_vmevent(struct event_t *e, char *eventid, char *data,
size_t dlen)
{
char *vmkey;
char *event;
char *type;
char *rest;
size_t klen;
size_t elen;
size_t tlen;
size_t rlen;
char *vmlogpath = NULL;
char *class;
char *files[512];
int count;
int i;
uint32_t severity;
char *log;
int res;
struct vm_event_t *vme;
vmkey = strings_ind(data, dlen, 0, &klen);
event = strings_ind(data, dlen, 1, &elen);
type = strings_ind(data, dlen, 2, &tlen);
rest = strings_ind(data, dlen, 3, &rlen);
if (!vmkey || !event || !type || !rest)
return;
if (strcmp(event, "CRASH") == 0)
severity = CRASH_SEVERITY;
else
severity = INFO_SEVERITY;
/* if line contains log, fill vmlogpath */
log = strstr(rest, ANDROID_LOGS_DIR);
if (log) {
const char *logf;
if (!e->dir)
return;
logf = log + sizeof(ANDROID_LOGS_DIR) - 1;
if (asprintf(&vmlogpath, "%s/%s", e->dir, logf) == -1)
return;
}
vme = (struct vm_event_t *)e->private;
res = asprintf(&class, "%s/%s/%s", vme->vm->name, event, type);
if (res < 0) {
LOGE("compute string failed, out of memory\n");
goto free_vmlogpath;
}
if (!vmlogpath) {
telemd_send_data("vm event doesn't contain logs", eventid,
severity, class);
goto free_class;
}
/* send logs */
count = lsdir(vmlogpath, files, ARRAY_SIZE(files));
if (count > 2) {
for (i = 0; i < count; i++) {
if (!strstr(files[i], "/.") &&
!strstr(files[i], "/..")) {
telemd_send_data(files[i], eventid, severity,
class);
}
}
} else if (count < 0) {
LOGE("lsdir (%s) failed, error (%s)\n", vmlogpath,
strerror(-count));
} else {
LOGE("get (%d) files in (%s) ???\n", count, vmlogpath);
}
while (count > 0)
free(files[--count]);
free_class:
free(class);
free_vmlogpath:
if (vmlogpath)
free(vmlogpath);
return;
}
static int telemd_new_event(struct event_t *e, char *result, size_t rsize,
char **eid)
{
struct crash_t *crash;
struct info_t *info;
struct vm_event_t *vme;
char *key;
const char *e_subtype = NULL;
size_t e_subtype_len = 0;
const char *estr = etype_str[e->event_type];
size_t eslen = strlen(etype_str[e->event_type]);
switch (e->event_type) {
case CRASH:
crash = (struct crash_t *)e->private;
e_subtype = crash->name;
e_subtype_len = crash->name_len;
break;
case INFO:
info = (struct info_t *)e->private;
e_subtype = info->name;
e_subtype_len = info->name_len;
break;
case UPTIME:
return 0;
case REBOOT:
break;
case VM:
vme = (struct vm_event_t *)e->private;
estr = vme->vm->name;
eslen = vme->vm->name_len;
e_subtype = strings_ind(result, rsize, 2, &e_subtype_len);
if (!e_subtype)
return -1;
break;
default:
break;
}
key = generate_event_id(estr, eslen, e_subtype, e_subtype_len,
KEY_LONG);
if (!key) {
LOGE("failed to generate event id, %s\n", strerror(errno));
return -1;
}
*eid = key;
return 0;
}
static int telemd_event_analyze(struct event_t *e, char **result,
size_t *rsize)
{
if (e->event_type == VM) {
struct vm_event_t *vme = (struct vm_event_t *)e->private;
if (android_event_analyze(vme->vm_msg, vme->vm_msg_len,
result, rsize) == -1) {
LOGE("failed to analyze android event\n");
return -1;
}
}
return 0;
}
static void telemd_send(struct event_t *e)
{
size_t rsize;
char *result = NULL;
char *eid = NULL;
if (telemd_event_analyze(e, &result, &rsize) == -1) {
LOGE("failed to analyze event\n");
return;
}
if (telemd_new_event(e, result, rsize, &eid) == -1) {
LOGE("failed to request resouce\n");
if (result)
free(result);
return;
}
switch (e->event_type) {
case CRASH:
telemd_send_crash(e, eid);
break;
case INFO:
telemd_send_info(e, eid);
break;
case UPTIME:
telemd_send_uptime();
break;
case REBOOT:
telemd_send_reboot(eid);
break;
case VM:
telemd_send_vmevent(e, eid, result, rsize);
break;
default:
LOGE("unsupoorted event type %d\n", e->event_type);
}
if (eid)
free(eid);
if (result)
free(result);
}
#endif
static void crashlog_send_crash(struct event_t *e, char *eid,
char *data, size_t dlen)
{
@ -1110,11 +696,6 @@ int init_sender(void)
LOGE("failed to init outdir size\n");
return -1;
}
#ifdef HAVE_TELEMETRICS_CLIENT
} else if (!strcmp(sender->name, "telemd")) {
sender->send = telemd_send;
#endif
}
}

View File

@ -1,2 +0,0 @@
#Type Path Mode UID GID Age Argument
d /var/log/crashlog 0750 telemetry telemetry -

View File

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (C) <2018> Intel Corporation
# Copyright (C) 2021 Intel Corporation
# SPDX-License-Identifier: BSD-3-Clause
#
# crashlogctl is part of acrn-hypervisor.
@ -11,37 +11,14 @@ declare -a CRASHLOG_SERVICES=(
usercrash.service
)
declare -a TELEMD_SERVICES=(
hprobe.timer
telemd-update-trigger.service
pstore-clean.service
pstore-probe.service
oops-probe.service
klogscanner.service
journal-probe.service
bert-probe.service
)
SCRIPT="$0"
TELEM_DIR=/etc/telemetrics
OPT_OUT_FILE=${TELEM_DIR}/opt-out
USER_CONF_FILE=${TELEM_DIR}/telemetrics.conf
SYSTEM_CONF_FILE=/usr/share/defaults/telemetrics/telemetrics.conf
CRASHLOG_SHARE_DIR=/usr/share/acrn/crashlog
CRASHLOG_SYSTEM_CONF=${CRASHLOG_SHARE_DIR}/40-watchdog.conf
CRASHLOG_SYSCTL_CONF=${CRASHLOG_SHARE_DIR}/80-coredump.conf
CRASHLOG_VAR_DIR=/var/log/crashlog
CRASHLOG_CORE_BACKUP=${CRASHLOG_VAR_DIR}/default_core_pattern
CRASHLOG_S_D_BACKUP=${CRASHLOG_VAR_DIR}/server_delivery_enabled
CRASHLOG_R_R_BACKUP=${CRASHLOG_VAR_DIR}/record_retention_enabled
CRASHLOG_WRK_DIRS_CONF=/usr/lib/tmpfiles.d/acrn-crashlog-dirs.conf
CORE_PATTERN_CONF="/proc/sys/kernel/core_pattern"
create_work_dirs() {
# Creates dirs if missing, adjust ownership if exists
systemd-tmpfiles --create ${CRASHLOG_WRK_DIRS_CONF}
}
exit_ok() {
echo "$1" > /dev/stderr
exit 0
@ -66,9 +43,6 @@ for_each_service() {
}
crashlog_enable() {
[ -f $OPT_OUT_FILE ] && exit_err "Opt out is enabled. Cannot start services."
# trigger systemd-tmpfiles work dirs creation
create_work_dirs
# backup the default core_pattern
if [ -f ${CRASHLOG_CORE_BACKUP} ]
@ -79,36 +53,12 @@ crashlog_enable() {
notice "... Backup core pattern to ${CRASHLOG_CORE_BACKUP}"
fi
# copy the configure file
if [ ! -f ${USER_CONF_FILE} ]
then
mkdir -p ${TELEM_DIR}
cp -v ${SYSTEM_CONF_FILE} ${USER_CONF_FILE}
fi
# modify the telemetics configure file
if grep --quiet server_delivery_enabled=true ${USER_CONF_FILE}
then
sed -i "s/server_delivery_enabled=true/server_delivery_enabled=false/g" ${USER_CONF_FILE}
echo 1 > ${CRASHLOG_S_D_BACKUP}
notice "... Set server_delivery_enabled=false in ${USER_CONF_FILE}"
fi
if grep --quiet record_retention_enabled=false ${USER_CONF_FILE}
then
sed -i "s/record_retention_enabled=false/record_retention_enabled=true/g" ${USER_CONF_FILE}
echo 1 > ${CRASHLOG_R_R_BACKUP}
notice "... Set record_retention_enabled=true in ${USER_CONF_FILE}"
fi
# Copy watchdog and coredump conf files
mkdir -p /etc/systemd/system.conf.d
cp -v ${CRASHLOG_SYSTEM_CONF} /etc/systemd/system.conf.d
mkdir -p /etc/sysctl.d
cp -v ${CRASHLOG_SYSCTL_CONF} /etc/sysctl.d
# Mask telemd services
for_each_service "mask" ${TELEMD_SERVICES[@]}
# Enable chrashlog services
for_each_service "enable" ${CRASHLOG_SERVICES[@]}
@ -118,27 +68,10 @@ crashlog_enable() {
crashlog_disable() {
# Disable chrashlog services
for_each_service "disable" ${CRASHLOG_SERVICES[@]}
# Unmask telemd services
for_each_service "unmask" ${TELEMD_SERVICES[@]}
rm -v /etc/sysctl.d/${CRASHLOG_SYSCTL_CONF##*/}
rm -v /etc/systemd/system.conf.d/${CRASHLOG_SYSTEM_CONF##*/}
# modify the telemetics configure file
if [ -f ${CRASHLOG_S_D_BACKUP} ]
then
sed -i "s/server_delivery_enabled=false/server_delivery_enabled=true/g" ${USER_CONF_FILE}
rm -f ${CRASHLOG_S_D_BACKUP}
notice "... Set server_delivery_enabled=true in ${USER_CONF_FILE}"
fi
if [ -f ${CRASHLOG_R_R_BACKUP} ]
then
sed -i "s/record_retention_enabled=true/record_retention_enabled=false/g" ${USER_CONF_FILE}
rm -f ${CRASHLOG_R_R_BACKUP}
notice "... Set record_retention_enabled=false in ${USER_CONF_FILE}"
fi
rm -f ${CRASHLOG_CORE_BACKUP}
exit_ok "*** Please reboot your system. ***"
@ -146,8 +79,6 @@ crashlog_disable() {
crashlog_is_active() {
# check only activation units
echo "telemprobd :" $(systemctl is-active telemprobd.socket)
echo "telempostd :" $(systemctl is-active telempostd.path)
echo "acrnprobe :" $(systemctl is-active acrnprobe.service)
echo "usercrash :" $(systemctl is-active usercrash.service)
}

View File

@ -1,8 +0,0 @@
digraph {
bgcolor=transparent; rankdir=LR;
node [shape="rectangle" style="filled" color="lightblue"]
edge [fontsize="12" fontcolor="blue"]
"acrnprobe" -> "telemetrics-client" [label="crashlog\npath"]
"telemetrics-client" -> "backend" [label="log\ncontent"]
}