From af9b3ad7a2c0bded9813886b0025e87acea862f6 Mon Sep 17 00:00:00 2001 From: David Scott Date: Tue, 29 Mar 2016 21:20:53 +0100 Subject: [PATCH 1/8] Allow diagnostics to be downloaded on port 62374 A client like `pinata diagnose` can do the equivalent of: nc 192.168.64.2 62374 > foo.tar and find `foo.tar` contains - the output of `/usr/bin/diagnostics` - `iptables -L` - `netstat -an` - `ps uax` - `docker ps` - `dig docker.com` - `wget http://docker.com` This will allow us to diagnose various in-VM faults, without having to ask the user to access the console and run commands. Signed-off-by: David Scott --- alpine/Dockerfile | 2 ++ .../packages/diagnostics/diagnostics-download | 20 +++++++++++++++++++ .../packages/diagnostics/diagnostics-server | 7 +++++++ .../diagnostics/etc/init.d/diagnostics | 1 + 4 files changed, 30 insertions(+) create mode 100755 alpine/packages/diagnostics/diagnostics-download create mode 100755 alpine/packages/diagnostics/diagnostics-server diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 9e2ad2c8c..4dd4dff4b 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -34,6 +34,8 @@ COPY packages/docker/docker /usr/bin/ COPY packages/docker/etc /etc/ COPY packages/docker-x/docker-x /usr/bin/ COPY packages/diagnostics/diagnostics /usr/bin/ +COPY packages/diagnostics/diagnostics-server /usr/bin/ +COPY packages/diagnostics/diagnostics-download /usr/bin/ COPY packages/diagnostics/etc /etc/ COPY packages/automount/etc /etc/ COPY packages/9pinit/etc /etc/ diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download new file mode 100755 index 000000000..e859edd3d --- /dev/null +++ b/alpine/packages/diagnostics/diagnostics-download @@ -0,0 +1,20 @@ +#!/bin/sh + +# Gather diagnostic data and write a .tar file to stdout + +TEMP=$(mktemp -d diagnoseXXXXXXX) +trap 'rm -rf "$TEMP"' EXIT +cd $TEMP + +# gather diagnostic data +ps uax > "ps -aux" +netstat -an > "netstat -an" +iptables -L > "iptables -L" +dmesg > dmesg +timeout -t 2 docker ps > "docker ps" +/usr/bin/diagnostics > "diagnostics" +dig docker.com > "dig docker.com" +wget -O - http://www.docker.com/ &> "wget docker.com" + +# send everything to the client +tar -c . diff --git a/alpine/packages/diagnostics/diagnostics-server b/alpine/packages/diagnostics/diagnostics-server new file mode 100755 index 000000000..611b0f28c --- /dev/null +++ b/alpine/packages/diagnostics/diagnostics-server @@ -0,0 +1,7 @@ +#!/bin/sh + +# Allow clients (e.g. pinata diagnose) to download diagnostic data + +while /bin/true; do + nc -l -p 62374 -e /usr/bin/diagnostics-download 2>> /var/log/diagnostics-server.log +done diff --git a/alpine/packages/diagnostics/etc/init.d/diagnostics b/alpine/packages/diagnostics/etc/init.d/diagnostics index 2629bfeb6..a5cbf056a 100755 --- a/alpine/packages/diagnostics/etc/init.d/diagnostics +++ b/alpine/packages/diagnostics/etc/init.d/diagnostics @@ -10,4 +10,5 @@ start() ebegin "Checking system state" /usr/bin/diagnostics + /usr/bin/diagnostics-server & } From 75f60eca3cb58833f8c7fda702934237a433f87e Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 30 Mar 2016 10:34:04 +0100 Subject: [PATCH 2/8] diagnostics-server: add more diagnostics For networking: - ifconfig -a - brctl show For storage: - mount - df Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index e859edd3d..1d5084819 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -10,8 +10,12 @@ cd $TEMP ps uax > "ps -aux" netstat -an > "netstat -an" iptables -L > "iptables -L" +ifconfig -a > "ifconfig -a" +brctl show > "brctl show" dmesg > dmesg timeout -t 2 docker ps > "docker ps" +mount > "mount" +df > "df" /usr/bin/diagnostics > "diagnostics" dig docker.com > "dig docker.com" wget -O - http://www.docker.com/ &> "wget docker.com" From a97038273eb882d50bc88de522290df2f90e7348 Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 30 Mar 2016 14:25:58 +0100 Subject: [PATCH 3/8] diagnose: try a ping, grab /etc/resolv.conf and `route -n` Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index 1d5084819..9206274e8 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -11,12 +11,15 @@ ps uax > "ps -aux" netstat -an > "netstat -an" iptables -L > "iptables -L" ifconfig -a > "ifconfig -a" +route -n > "route -n" brctl show > "brctl show" dmesg > dmesg timeout -t 2 docker ps > "docker ps" mount > "mount" df > "df" /usr/bin/diagnostics > "diagnostics" +ping -w 5 8.8.8.8 &> "ping -w 5 8.8.8.8" +cp /etc/resolv.conf . dig docker.com > "dig docker.com" wget -O - http://www.docker.com/ &> "wget docker.com" From d24dd9d59bf3d4f874dcc49f2d34c3447116c5fa Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 30 Mar 2016 14:28:00 +0100 Subject: [PATCH 4/8] diagnose: ls -l /var, /var/lib and /var/lib/docker Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index 9206274e8..3287a878a 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -17,6 +17,9 @@ dmesg > dmesg timeout -t 2 docker ps > "docker ps" mount > "mount" df > "df" +ls -l /var &> "ls -l var" +ls -l /var/lib &> "ls -l var_lib" +ls -l /var/lib/docker &> "ls -l var_lib_docker" /usr/bin/diagnostics > "diagnostics" ping -w 5 8.8.8.8 &> "ping -w 5 8.8.8.8" cp /etc/resolv.conf . From f9eca403715171d17de100534347e44f2f63103f Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 30 Mar 2016 14:31:20 +0100 Subject: [PATCH 5/8] diagnose: put all the files inside a moby/ subdirectory Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index 3287a878a..9549c9d61 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -4,9 +4,12 @@ TEMP=$(mktemp -d diagnoseXXXXXXX) trap 'rm -rf "$TEMP"' EXIT -cd $TEMP +mkdir $TEMP/moby +cd $TEMP/moby # gather diagnostic data +date > "date" +uname -a > "uname -a" ps uax > "ps -aux" netstat -an > "netstat -an" iptables -L > "iptables -L" @@ -27,4 +30,5 @@ dig docker.com > "dig docker.com" wget -O - http://www.docker.com/ &> "wget docker.com" # send everything to the client +cd .. tar -c . From 82e9dfc4eec1c48bcc0a497404ded9117bebdd68 Mon Sep 17 00:00:00 2001 From: David Scott Date: Wed, 30 Mar 2016 15:42:48 +0100 Subject: [PATCH 6/8] diagnose: grab the `tail /var/log/docker.log` Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index 9549c9d61..fe39a37dd 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -18,6 +18,7 @@ route -n > "route -n" brctl show > "brctl show" dmesg > dmesg timeout -t 2 docker ps > "docker ps" +tail /var/log/docker.log > "docker.log" mount > "mount" df > "df" ls -l /var &> "ls -l var" From 157d2aef07420661c694254d2e9857fa65bab5a2 Mon Sep 17 00:00:00 2001 From: David Scott Date: Thu, 31 Mar 2016 10:35:35 +0100 Subject: [PATCH 7/8] diagnostics: use `netstat -tulpn` which includes process names For example: docker:~# netstat -tulpn Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 :::32768 :::* LISTEN 1344/docker-proxy tcp 0 0 :::2375 :::* LISTEN 1124/docker tcp 0 0 :::5000 :::* LISTEN 1444/docker-proxy tcp 0 0 :::5001 :::* LISTEN 1494/docker-proxy Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index fe39a37dd..7d3a8bb2d 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -11,7 +11,7 @@ cd $TEMP/moby date > "date" uname -a > "uname -a" ps uax > "ps -aux" -netstat -an > "netstat -an" +netstat -tulpn > "netstat -tulpn" iptables -L > "iptables -L" ifconfig -a > "ifconfig -a" route -n > "route -n" From 1beef9a6c8abe999c8b64475c076b10ef6a4dcfd Mon Sep 17 00:00:00 2001 From: David Scott Date: Thu, 31 Mar 2016 11:14:31 +0100 Subject: [PATCH 8/8] diagnose: include `iptables -t nat -L` The extra `-t nat` includes the masquerading stuff. Signed-off-by: David Scott --- alpine/packages/diagnostics/diagnostics-download | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine/packages/diagnostics/diagnostics-download b/alpine/packages/diagnostics/diagnostics-download index 7d3a8bb2d..559a10689 100755 --- a/alpine/packages/diagnostics/diagnostics-download +++ b/alpine/packages/diagnostics/diagnostics-download @@ -12,7 +12,7 @@ date > "date" uname -a > "uname -a" ps uax > "ps -aux" netstat -tulpn > "netstat -tulpn" -iptables -L > "iptables -L" +iptables -t nat -L > "iptables -t nat -L" ifconfig -a > "ifconfig -a" route -n > "route -n" brctl show > "brctl show"