tools:acrn-crashlog: detect the panic event from all pstore files

As all pstore files may include the kernel panic information, this patch
tries to find the clue from all messages under /sys/fs/pstore instead of
console-ramoops-0.
To avoid the constant growing of logs, it has to remove the pstore
files after panic being detected.

Tracked-On: #3390
Signed-off-by: Liu, Xinwu <xinwu.liu@intel.com>
Reviewed-by: Zhi Jin <zhi.jin@intel.com>
Acked-by: Chen, Gang <gang.c.chen@intel.com>

tools/acrn-crashlog/acrnprobe/channels.c

@@ -24,6 +24,7 @@
 #include "probeutils.h"
 #include "log_sys.h"
 #include "android_events.h"
+#include "crash_reclassify.h"
 
 #define POLLING_TIMER_SIG 0xCEAC
 
@@ -122,14 +123,16 @@ static void channel_oneshot(struct channel_t *cnl)
 		if (!crash->trigger)
 			continue;
 
-		if (!strcmp("file", crash->trigger->type)) {
-			if (file_exists(crash->trigger->path)) {
-				e = create_event(CRASH, cname, (void *)crash,
-						 0, crash->trigger->path,
-						 crash->trigger->path_len);
-				if (e)
-					event_enqueue(e);
-			}
+		if (!strcmp("file", crash->trigger->type) ||
+				!strcmp("node", crash->trigger->type)) {
+			if (!crash_match_filefmt(crash, crash->trigger->path))
+				continue;
+
+			e = create_event(CRASH, cname, (void *)crash,
+					 0, crash->trigger->path,
+					 crash->trigger->path_len);
+			if (e)
+				event_enqueue(e);
 		} else if (!strcmp("rebootreason", crash->trigger->type)) {
 			char rreason[REBOOT_REASON_SIZE];
 

tools/acrn-crashlog/acrnprobe/crash_reclassify.c

@@ -209,18 +209,53 @@ fail:
 	return -1;
 }
 
+static int crash_match_file(const struct crash_t *crash, const char *filename)
+{
+	size_t size;
+	void *cnt;
+
+	if (read_file(filename, &size, &cnt) == -1) {
+		LOGE("read %s failed, error (%s)\n", filename, strerror(errno));
+		return 0;
+	}
+	if (!size)
+		return 0;
+	if (crash_match_content(crash, cnt)) {
+		free(cnt);
+		return 1;
+	}
+	free(cnt);
+	return 0;
+}
+
+int crash_match_filefmt(const struct crash_t *crash, const char *filefmt)
+{
+	int count;
+	int i;
+	int ret = 0;
+	char **files;
+
+	count = config_fmt_to_files(filefmt, &files);
+	if (count <= 0)
+		return ret;
+	for (i = 0; i < count; i++) {
+		if (crash_match_file(crash, files[i])) {
+			ret = 1;
+			break;
+		}
+	}
+	for (i = 0; i < count; i++)
+		free(files[i]);
+	free(files);
+	return ret;
+}
+
 static struct crash_t *crash_find_matched_child(const struct crash_t *crash,
 						const char *rtrfmt)
 {
 	struct crash_t *child;
 	struct crash_t *matched_child = NULL;
-	int i;
-	int count;
-	int res;
 	const char *trfile_fmt;
-	char **trfiles;
-	void *content;
-	unsigned long size;
 
 	if (!crash)
 		return NULL;
@@ -234,33 +269,10 @@ static struct crash_t *crash_find_matched_child(const struct crash_t *crash,
 		else
 			trfile_fmt = child->trigger->path;
 
-		count = config_fmt_to_files(trfile_fmt, &trfiles);
-		if (count <= 0)
-			continue;
-
-		for (i = 0; i < count; i++) {
-			res = read_file(trfiles[i], &size, &content);
-			if (res == -1) {
-				LOGE("read %s failed, error (%s)\n",
-				     trfiles[i], strerror(errno));
-				continue;
-			}
-			if (!size)
-				continue;
-			if (crash_match_content(child, content)) {
-				free(content);
-				matched_child = child;
-				break;
-			}
-			free(content);
-		}
-
-		for (i = 0; i < count; i++)
-			free(trfiles[i]);
-		free(trfiles);
-
-		if (matched_child)
+		if (crash_match_filefmt(child, trfile_fmt)) {
+			matched_child = child;
 			break;
+		}
 	}
 
 	/* It returns the first matched crash */

tools/acrn-crashlog/acrnprobe/include/crash_reclassify.h

@@ -3,4 +3,6 @@
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
+extern int crash_match_filefmt(const struct crash_t *crash,
+				const char *filefmt);
 extern void init_crash_reclassify(void);

tools/acrn-crashlog/acrnprobe/include/load_conf.h

@@ -61,6 +61,8 @@ struct log_t {
 	size_t		path_len;
 	const char	*lines;
 	size_t		lines_len;
+	const char	*deletesource;
+	size_t		deletesource_len;
 	const char	*sizelimit;
 	size_t		sizelimit_len;
 

tools/acrn-crashlog/acrnprobe/load_conf.c

@@ -71,6 +71,7 @@ static void print(void)
 			continue;
 		print_id_item(name, log, id);
 		print_id_item(type, log, id);
+		print_id_item(deletesource, log, id);
 		print_id_item(lines, log, id);
 		print_id_item(path, log, id);
 		print_id_item(sizelimit, log, id);
@@ -447,6 +448,8 @@ static int parse_log(xmlNodePtr cur, struct log_t *log)
 			res = load_cur_content(cur, log, name);
 		else if (name_is(cur, "type"))
 			res = load_cur_content(cur, log, type);
+		else if (name_is(cur, "deletesource"))
+			res = load_cur_content(cur, log, deletesource);
 		else if (name_is(cur, "path"))
 			res = load_cur_content(cur, log, path);
 		else if (name_is(cur, "lines"))

tools/acrn-crashlog/acrnprobe/sender.c

@@ -31,12 +31,6 @@
 #define CRASH_SEVERITY 4
 #define INFO_SEVERITY 2
 
-struct telemd_data_t {
-	char *class;
-	char *srcdir;
-	char *eventid;
-	uint32_t severity;
-};
 #endif
 
 static int crashlog_check_space(void)
@@ -214,10 +208,13 @@ static void get_log_by_type(const char *despath, const struct log_t *log,
 			if (cfg_atoi(log->sizelimit, log->sizelimit_len,
 				     &size) == -1)
 				return;
-		get_log_node(despath, log->path, (size_t)(size * 1024 * 1024));
+		get_log_node(despath, srcpath, (size_t)(size * 1024 * 1024));
 	}
 	else if (!strcmp("cmd", log->type))
-		get_log_cmd(despath, log->path);
+		get_log_cmd(despath, srcpath);
+
+	if (log->deletesource && !strcmp("true", log->deletesource))
+		remove(srcpath);
 }
 #ifdef HAVE_TELEMETRICS_CLIENT
 static int telemd_send_data(char *payload, char *eventid, uint32_t severity,
@@ -264,56 +261,45 @@ fail:
 	return -1;
 }
 
-static void telemd_get_log(struct log_t *log, void *data)
+static void telemd_get_log(const char *dir, char *eventid,
+				uint32_t severity, char *class)
 {
-	const struct telemd_data_t *d = (struct telemd_data_t *)data;
-	char fpath[PATH_MAX];
+	char *files[512];
 	char *msg;
 	int count;
-	int len;
 	int i;
-	struct dirent **filelist;
-	struct ac_filter_data acfd = {log->name, log->name_len};
 
-	if (d->srcdir == NULL)
+	if (!dir)
 		goto send_nologs;
 
-	/* search file which use log->name as substring */
-	count = ac_scandir(d->srcdir, &filelist, filter_filename_substr,
-			   &acfd, NULL);
+	/* send logs */
+	count = lsdir(dir, files, ARRAY_SIZE(files));
 	if (count < 0) {
-		LOGE("error occurs when scanning (%s)\n", d->srcdir);
+		LOGE("lsdir (%s) failed, error (%s)\n", dir, strerror(-count));
 		return;
 	}
-	if (!count) {
-		LOGE("couldn't find any files with substr (%s) under (%s)\n",
-		     log->name, d->srcdir);
-		goto send_nologs;
+
+	if (count > 2) {
+		for (i = 0; i < count; i++) {
+			if (strstr(files[i], "/.") || strstr(files[i], "/.."))
+				continue;
+			telemd_send_data(files[i], eventid, severity, class);
+		}
+		while (count > 0)
+			free(files[--count]);
+		return;
+	} else if (count == 2) {
+		while (count > 0)
+			free(files[--count]);
 	}
 
-	for (i = 0; i < count; i++) {
-		len = snprintf(fpath, sizeof(fpath), "%s/%s", d->srcdir,
-			       filelist[i]->d_name);
-		free(filelist[i]);
-		if (s_not_expect(len, sizeof(fpath)))
-			LOGW("failed to generate path, event %s\n", d->eventid);
-		else
-			telemd_send_data(fpath, d->eventid,
-					 d->severity, d->class);
-	}
-
-	free(filelist);
-
-	return;
-
 send_nologs:
-	if (asprintf(&msg, "couldn't find logs with (%s), check probe's log.",
-		     log->name) == -1) {
+	if (asprintf(&msg, "no logs provided, check probe's log.") == -1) {
 		LOGE("failed to generate msg, out of memory\n");
 		return;
 	}
 
-	telemd_send_data(msg, d->eventid, d->severity, d->class);
+	telemd_send_data(msg, eventid, severity, class);
 	free(msg);
 }
 #endif
@@ -387,14 +373,8 @@ static void crashlog_get_log(struct log_t *log, void *data)
 static void telemd_send_crash(struct event_t *e, char *eventid)
 {
 	struct crash_t *crash;
-	struct log_t *log;
 	char *class;
-	int id;
 	int ret;
-	struct telemd_data_t data = {
-		.srcdir = e->dir,
-		.severity = CRASH_SEVERITY,
-	};
 
 	crash = (struct crash_t *)e->private;
 
@@ -404,68 +384,15 @@ static void telemd_send_crash(struct event_t *e, char *eventid)
 		return;
 	}
 
-	data.class = class;
-	data.eventid = eventid;
-
-	for_each_log_collect(id, log, crash) {
-		if (!log)
-			continue;
-
-		log->get(log, (void *)&data);
-	}
-	if (!strcmp(e->channel, "inotify")) {
-		char *des;
-		/* get the trigger file */
-		ret = asprintf(&des, "%s/%s", e->dir, e->path);
-		if (ret < 0) {
-			LOGE("compute string failed, out of memory\n");
-			goto free_class;
-		}
-
-		if (!file_exists(des)) {
-			/* find the original path */
-			char *ori;
-
-			ret = asprintf(&ori, "%s/%s", crash->trigger->path,
-				       e->path);
-			if (ret < 0) {
-				LOGE("compute string failed, out of memory\n");
-				free(des);
-				goto free_class;
-			}
-
-			LOGW("(%s) unavailable, try the original path (%s)\n",
-			     des, ori);
-			if (!file_exists(ori)) {
-				LOGE("original path (%s) is unavailable\n",
-				     ori);
-			} else {
-				telemd_send_data(ori, eventid, CRASH_SEVERITY,
-						 class);
-			}
-
-			free(ori);
-		} else {
-			telemd_send_data(des, eventid, CRASH_SEVERITY, class);
-		}
-
-		free(des);
-	}
-free_class:
+	telemd_get_log(e->dir, eventid, CRASH_SEVERITY, class);
 	free(class);
 }
 
 static void telemd_send_info(struct event_t *e, char *eventid)
 {
 	struct info_t *info;
-	struct log_t *log;
 	char *class;
-	int id;
 	int ret;
-	struct telemd_data_t data = {
-		.srcdir = e->dir,
-		.severity = INFO_SEVERITY,
-	};
 
 	info = (struct info_t *)e->private;
 	ret = asprintf(&class, "clearlinux/info/%s", info->name);
@@ -474,16 +401,7 @@ static void telemd_send_info(struct event_t *e, char *eventid)
 		return;
 	}
 
-	data.class = class;
-	data.eventid = eventid;
-
-	for_each_log_collect(id, log, info) {
-		if (!log)
-			continue;
-
-		log->get(log, (void *)&data);
-	}
-
+	telemd_get_log(e->dir, eventid, INFO_SEVERITY, class);
 	free(class);
 }
 
@@ -730,8 +648,6 @@ static int telemd_event_analyze(struct event_t *e, char **result,
 
 static void telemd_send(struct event_t *e)
 {
-	int id;
-	struct log_t *log;
 	size_t rsize;
 	char *result = NULL;
 	char *eid = NULL;
@@ -746,13 +662,6 @@ static void telemd_send(struct event_t *e)
 			free(result);
 		return;
 	}
-	for_each_log(id, log, conf) {
-		if (!log)
-			continue;
-
-		log->get = telemd_get_log;
-	}
-
 	switch (e->event_type) {
 	case CRASH:
 		telemd_send_crash(e, eid);

tools/acrn-crashlog/common/fsutils.c

@@ -842,6 +842,8 @@ int filter_filename_startswith(const struct dirent *entry,
 	if (_D_EXACT_NAMLEN(entry) < d->len)
 		return -1;
 
+	if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+		return -1;
 	return memcmp(entry->d_name, d->str, d->len);
 }
 

tools/acrn-crashlog/data/acrnprobe.xml

@@ -29,7 +29,7 @@
 		<trigger id="1" enable="true">
 			<name>t_pstore</name>
 			<type>node</type>
-			<path>/sys/fs/pstore/console-ramoops-0</path>
+			<path>/sys/fs/pstore/[*]</path>
 		</trigger>
 		<trigger id="2" enable="true">
 			<name>t_boot</name>
@@ -67,7 +67,8 @@
 		<log id="1" enable="true">
 			<name>pstore</name>
 			<type>node</type>
-			<path>/sys/fs/pstore/console-ramoops-0</path>
+			<deletesource>true</deletesource>
+			<path>/sys/fs/pstore/[*]</path>
 		</log>
 		<log id='2' enable='true'>
 			<name>kmsg</name>
@@ -160,6 +161,17 @@
 			<log id='1'>kmsg</log>
 			<log id='2'>syslog</log>
 		</crash>
+		<crash id='9' inherit='0' enable='true'>
+			<name>IPANIC</name>
+			<trigger>t_pstore</trigger>
+			<channel>oneshot</channel>
+			<mightcontent expression='1' id='1'>Kernel panic - not syncing:</mightcontent>
+			<mightcontent expression='1' id='2'>BUG: unable to handle kernel</mightcontent>
+			<data id='1'>kernel BUG at</data>
+			<data id='2'>EIP is at</data>
+			<data id='3'>Comm:</data>
+			<log id='1'>pstore</log>
+		</crash>
 	</crashes>
 
 	<infos>