From 0c471bdc09962ed5d2a975664a372c3c0d39b930 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 17 Jun 2016 11:56:57 +0100 Subject: [PATCH 01/10] Simple build system for ocaml-based iptables Signed-off-by: Ian Campbell --- alpine/Dockerfile | 1 + alpine/packages/Makefile | 2 ++ alpine/packages/iptables/Dockerfile | 5 +++++ alpine/packages/iptables/Makefile | 9 +++++++++ alpine/packages/iptables/main.ml | 2 ++ 5 files changed, 19 insertions(+) create mode 100644 alpine/packages/iptables/Dockerfile create mode 100644 alpine/packages/iptables/Makefile create mode 100644 alpine/packages/iptables/main.ml diff --git a/alpine/Dockerfile b/alpine/Dockerfile index b420df3de..1e085af99 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -75,6 +75,7 @@ COPY packages/9pmount-vsock/9pmount-vsock /sbin COPY packages/test/etc /etc COPY packages/test/mobytest /usr/bin COPY packages/sysctl/etc /etc +COPY packages/iptables/iptables /usr/local/sbin/iptables RUN \ rc-update add swap boot && \ diff --git a/alpine/packages/Makefile b/alpine/packages/Makefile index 727332528..0048273d0 100644 --- a/alpine/packages/Makefile +++ b/alpine/packages/Makefile @@ -10,6 +10,7 @@ all: $(MAKE) -C llmnrd OS=linux $(MAKE) -C gummiboot OS=linux $(MAKE) -C 9pmount-vsock OS=linux + $(MAKE) -C iptables OS=linux arm: $(MAKE) -C transfused OS=linux ARCH=arm @@ -33,3 +34,4 @@ clean: $(MAKE) -C llmnrd clean $(MAKE) -C gummiboot clean $(MAKE) -C 9pmount-vsock clean + $(MAKE) -C iptables clean diff --git a/alpine/packages/iptables/Dockerfile b/alpine/packages/iptables/Dockerfile new file mode 100644 index 000000000..92a67e69c --- /dev/null +++ b/alpine/packages/iptables/Dockerfile @@ -0,0 +1,5 @@ +FROM ocaml/opam:alpine +WORKDIR /app +ADD . /app +RUN sudo chown -R opam /app +RUN ocamlopt -o iptables main.ml diff --git a/alpine/packages/iptables/Makefile b/alpine/packages/iptables/Makefile new file mode 100644 index 000000000..df248cc0a --- /dev/null +++ b/alpine/packages/iptables/Makefile @@ -0,0 +1,9 @@ +all: iptables + +iptables: Dockerfile main.ml + docker build -t iptables:build . + docker run --rm iptables:build cat /app/iptables > iptables + chmod 755 iptables + +clean: + docker images -q iptables:build | xargs docker rmi -f || true diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml new file mode 100644 index 000000000..a496c79cf --- /dev/null +++ b/alpine/packages/iptables/main.ml @@ -0,0 +1,2 @@ +let () = + print_string "Hello world!\n";; From 80b234dd3e3586dc0e2eec52383d01682b42c25b Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 12:57:26 +0100 Subject: [PATCH 02/10] Add primitive iptables wrapper which can set up port forwards Signed-off-by: David Scott --- alpine/packages/iptables/.gitignore | 1 + alpine/packages/iptables/Dockerfile | 4 +- alpine/packages/iptables/main.ml | 84 ++++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 alpine/packages/iptables/.gitignore diff --git a/alpine/packages/iptables/.gitignore b/alpine/packages/iptables/.gitignore new file mode 100644 index 000000000..3457ccc49 --- /dev/null +++ b/alpine/packages/iptables/.gitignore @@ -0,0 +1 @@ +iptables diff --git a/alpine/packages/iptables/Dockerfile b/alpine/packages/iptables/Dockerfile index 92a67e69c..c39b33bfe 100644 --- a/alpine/packages/iptables/Dockerfile +++ b/alpine/packages/iptables/Dockerfile @@ -1,5 +1,7 @@ FROM ocaml/opam:alpine +RUN sudo apk add m4 +RUN opam install ocamlfind astring -y WORKDIR /app ADD . /app RUN sudo chown -R opam /app -RUN ocamlopt -o iptables main.ml +RUN opam config exec -- ocamlfind ocamlopt -package unix,astring -linkpkg -o iptables main.ml diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index a496c79cf..37517cce2 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -1,2 +1,82 @@ -let () = - print_string "Hello world!\n";; +(* ocamlfind ocamlopt -package unix,astring -linkpkg -o iptables iptables.ml *) + +(* +--wait -t nat -I DOCKER-INGRESS -p tcp --dport 80 -j DNAT --to-destination 172.18.0.2:80 +--wait -t nat -D DOCKER-INGRESS -p tcp --dport 80 -j DNAT --to-destination 172.18.0.2:80 +*) + +let _iptables = "/sbin/iptables" +let _proxy = "/usr/bin/docker-proxy" +let _pid_dir = "/var/run/service-port-opener" + +type port = { + proto: string; + dport: string; (* host port *) + ip: string; (* container ip *) + port: string; (* container port *) +} + +let log_fd = Unix.openfile "/var/run/log/service-port-opener.log" [ Unix.O_WRONLY; Unix.O_APPEND; Unix.O_CREAT ] 0o0644 + +let logf fmt = + Printf.ksprintf (fun s -> + let s = s ^ "\n" in + let rec loop ofs remaining = + if remaining > 0 then begin + let n = Unix.write log_fd s ofs remaining in + loop (ofs + n) (remaining - n) + end in + loop 0 (String.length s) + ) fmt + +let pid_filename { proto; dport; ip; port } = + Printf.sprintf "%s/%s.%s.%s.%s.pid" _pid_dir proto dport ip port + +let insert ({ proto; dport; ip; port } as p) = + let filename = pid_filename p in + logf "insert: creating a proxy for %s" filename; + let args = [ _proxy; "-proto"; proto; "-container-ip"; ip; "-container-port"; port; "-host-ip"; "0.0.0.0"; "-host-port"; dport; "-i" ] in + let pid = Unix.fork () in + if pid != 0 then begin + (* write pid to a file (not atomically) *) + let oc = open_out filename in + output_string oc (string_of_int pid); + close_out oc; + logf "binary = %s args = %s" _proxy (String.concat "; " args); + (try Unix.execv _proxy (Array.of_list args) with e -> logf "Failed with %s" (Printexc.to_string e)); + exit 1 + end + +let delete ({ proto; dport; ip; port } as p) = + let filename = pid_filename p in + logf "delete: removing a proxy for %s" filename; + (* read the pid from a file *) + try + let ic = open_in filename in + let pid = int_of_string (input_line ic) in + logf "Sending SIGTERM to %d" pid; + Unix.kill Sys.sigterm pid + with e -> + logf "delete: failed to remove proxy for %s: %s" filename (Printexc.to_string e); + raise e + +let parse_ip_port ip_port = match Astring.String.cut ~sep:":" ip_port with + | None -> + failwith ("Failed to parse :" ^ ip_port) + | Some (ip, port) -> + ip, port + +let _ = + ( try Unix.mkdir _pid_dir 0o0755 with Unix.Unix_error(Unix.EEXIST, _, _) -> () ); + logf "intercepted arguments [%s]" (String.concat "; " (Array.to_list Sys.argv)); + ( match Array.to_list Sys.argv with + | [ _; "--wait"; "-t"; "nat"; "-I"; "DOCKER-INGRESS"; "-p"; proto; "--dport"; dport; "-j"; "DNAT"; "--to-destination"; ip_port ] -> + let ip, port = parse_ip_port ip_port in + insert { proto; dport; ip; port } + | [ _; "--wait"; "-t"; "nat"; "-D"; "DOCKER-INGRESS"; "-p"; proto; "--dport"; dport; "-j"; "DNAT"; "--to-destination"; ip_port ] -> + let ip, port = parse_ip_port ip_port in + delete { proto; dport; ip; port } + | _ -> + () + ); + Unix.execv _iptables Sys.argv From dfb97863c9a8c687747f2833bc732dbe376054e6 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 12:59:25 +0100 Subject: [PATCH 03/10] /etc/init.d/docker: prepend /usr/local/sbin to the $PATH This is where the iptables wrapper lives. Signed-off-by: David Scott --- alpine/packages/docker/etc/init.d/docker | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpine/packages/docker/etc/init.d/docker b/alpine/packages/docker/etc/init.d/docker index e8d8984f7..a1f43538f 100755 --- a/alpine/packages/docker/etc/init.d/docker +++ b/alpine/packages/docker/etc/init.d/docker @@ -60,6 +60,9 @@ start() DOCKER_LOGFILE="/var/log/docker.log" + # Allow iptables to be overriden + export PATH=/usr/local/sbin:$PATH + start-stop-daemon --start --quiet \ --background \ --exec ${command} \ From a294b0f9b0a10c6f22aeb5497ff687f002f4dd9e Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 14:40:09 +0100 Subject: [PATCH 04/10] iptables: close all the fds we inherit from docker Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 37517cce2..31da8c929 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -35,16 +35,26 @@ let pid_filename { proto; dport; ip; port } = let insert ({ proto; dport; ip; port } as p) = let filename = pid_filename p in logf "insert: creating a proxy for %s" filename; - let args = [ _proxy; "-proto"; proto; "-container-ip"; ip; "-container-port"; port; "-host-ip"; "0.0.0.0"; "-host-port"; dport; "-i" ] in + let args = [ _proxy; "-proto"; proto; "-container-ip"; ip; "-container-port"; port; "-host-ip"; "0.0.0.0"; "-host-port"; dport; "-i"; "-no-local-ip" ] in let pid = Unix.fork () in - if pid != 0 then begin + if pid == 0 then begin + logf "binary = %s args = %s" _proxy (String.concat "; " args); + (* Close the vast number of fds I've inherited from docker *) + for i = 0 to 1023 do + let fd : Unix.file_descr = Obj.magic i in + try Unix.close fd with Unix.Unix_error(Unix.EBADF, _, _) -> () + done; + let null = Unix.openfile "/dev/null" [ Unix.O_RDWR ] 0 in + Unix.dup2 null Unix.stdin; + Unix.dup2 null Unix.stdout; + Unix.dup2 null Unix.stderr; + (try Unix.execv _proxy (Array.of_list args) with e -> logf "Failed with %s" (Printexc.to_string e)); + exit 1 + end else begin (* write pid to a file (not atomically) *) let oc = open_out filename in output_string oc (string_of_int pid); - close_out oc; - logf "binary = %s args = %s" _proxy (String.concat "; " args); - (try Unix.execv _proxy (Array.of_list args) with e -> logf "Failed with %s" (Printexc.to_string e)); - exit 1 + close_out oc end let delete ({ proto; dport; ip; port } as p) = From 3183d9c72ae2293e8e91969a982208a27880f875 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 14:40:30 +0100 Subject: [PATCH 05/10] iptables: get the `kill` arguments the right way round Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 31da8c929..23db29202 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -65,10 +65,10 @@ let delete ({ proto; dport; ip; port } as p) = let ic = open_in filename in let pid = int_of_string (input_line ic) in logf "Sending SIGTERM to %d" pid; - Unix.kill Sys.sigterm pid + Unix.kill pid Sys.sigterm with e -> logf "delete: failed to remove proxy for %s: %s" filename (Printexc.to_string e); - raise e + () let parse_ip_port ip_port = match Astring.String.cut ~sep:":" ip_port with | None -> From 3c6ad764618333938ef0f40d103a63a5c3037ba1 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 14:41:00 +0100 Subject: [PATCH 06/10] proxy: add a -no-local-ip option docker itself seems to bind to the port globally inside Moby, so we get an EADDRINUSE if we try to do it too. Signed-off-by: David Scott --- alpine/packages/proxy/one.go | 21 +++++++++++++++------ alpine/packages/proxy/proxy.go | 13 +++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/alpine/packages/proxy/one.go b/alpine/packages/proxy/one.go index b8f4e1730..e91aa30af 100644 --- a/alpine/packages/proxy/one.go +++ b/alpine/packages/proxy/one.go @@ -11,11 +11,16 @@ import ( ) func onePort() { - host, _, container := parseHostContainerAddrs() + host, _, container, localIP := parseHostContainerAddrs() - ipP, err := libproxy.NewIPProxy(host, container) - if err != nil { - sendError(err) + var ipP libproxy.Proxy + var err error + + if localIP { + ipP, err = libproxy.NewIPProxy(host, container) + if err != nil { + sendError(err) + } } ctl, err := exposePort(host, container) @@ -23,10 +28,14 @@ func onePort() { sendError(err) } - go handleStopSignals(ipP) + go handleStopSignals() // TODO: avoid this line if we are running in a TTY sendOK() - ipP.Run() + if ipP != nil { + ipP.Run() + } else { + select{} // sleep forever + } ctl.Close() // ensure ctl remains alive and un-GCed until here os.Exit(0) } diff --git a/alpine/packages/proxy/proxy.go b/alpine/packages/proxy/proxy.go index c89ff8db2..adc3b8191 100644 --- a/alpine/packages/proxy/proxy.go +++ b/alpine/packages/proxy/proxy.go @@ -8,8 +8,6 @@ import ( "os" "os/signal" "syscall" - - "proxy/libproxy" ) var interactiveMode bool @@ -48,7 +46,7 @@ var vSockUDPPortOffset = 0x20000 // parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP // net.Addrs to map the host and container ports -func parseHostContainerAddrs() (host net.Addr, port int, container net.Addr) { +func parseHostContainerAddrs() (host net.Addr, port int, container net.Addr, localIP bool) { var ( proto = flag.String("proto", "tcp", "proxy protocol") hostIP = flag.String("host-ip", "", "host ip") @@ -56,6 +54,7 @@ func parseHostContainerAddrs() (host net.Addr, port int, container net.Addr) { containerIP = flag.String("container-ip", "", "container ip") containerPort = flag.Int("container-port", -1, "container port") interactive = flag.Bool("i", false, "print success/failure to stdout/stderr") + noLocalIP = flag.Bool("no-local-ip", false, "bind only on the Host, not in the VM") ) flag.Parse() @@ -73,17 +72,15 @@ func parseHostContainerAddrs() (host net.Addr, port int, container net.Addr) { default: log.Fatalf("unsupported protocol %s", *proto) } - - return host, port, container + localIP = ! *noLocalIP + return host, port, container, localIP } -func handleStopSignals(p libproxy.Proxy) { +func handleStopSignals() { s := make(chan os.Signal, 10) signal.Notify(s, os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP) for range s { os.Exit(0) - // The vsock proxy cannot be shutdown the same way as the TCP one: - //p.Close() } } From 12fbe114f675a11c309a268a8c7cd5779ddfbf63 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 15:52:24 +0100 Subject: [PATCH 07/10] iptables: remove the pid file after sending SIGTERM Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 23db29202..40f769669 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -65,7 +65,8 @@ let delete ({ proto; dport; ip; port } as p) = let ic = open_in filename in let pid = int_of_string (input_line ic) in logf "Sending SIGTERM to %d" pid; - Unix.kill pid Sys.sigterm + Unix.kill pid Sys.sigterm; + Unix.unlink filename with e -> logf "delete: failed to remove proxy for %s: %s" filename (Printexc.to_string e); () From d0876fb05eadcd07d48237ef371a7d0d74f584d1 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 15:53:10 +0100 Subject: [PATCH 08/10] iptables: add a TODO Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 40f769669..5bf1e07a3 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -40,6 +40,7 @@ let insert ({ proto; dport; ip; port } as p) = if pid == 0 then begin logf "binary = %s args = %s" _proxy (String.concat "; " args); (* Close the vast number of fds I've inherited from docker *) + (* TODO(djs55): revisit, possibly by filing a docker/docker issue *) for i = 0 to 1023 do let fd : Unix.file_descr = Obj.magic i in try Unix.close fd with Unix.Unix_error(Unix.EBADF, _, _) -> () From 55a2becfb4dd4b8b748c6806886af65ddcdafc6f Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 16:00:25 +0100 Subject: [PATCH 09/10] iptables: only open host ports if native/port-forwarding=true in the db Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 5bf1e07a3..323f334af 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -80,8 +80,14 @@ let parse_ip_port ip_port = match Astring.String.cut ~sep:":" ip_port with let _ = ( try Unix.mkdir _pid_dir 0o0755 with Unix.Unix_error(Unix.EEXIST, _, _) -> () ); - logf "intercepted arguments [%s]" (String.concat "; " (Array.to_list Sys.argv)); - ( match Array.to_list Sys.argv with + let port_forwarding = + try + let ic = open_in "/Database/branch/master/ro/com.docker.driver.amd64-linux/native/port-forwarding" in + bool_of_string (String.trim (input_line ic)) + with _ -> false in + logf "port_forwarding=%b intercepted arguments [%s]" port_forwarding (String.concat "; " (Array.to_list Sys.argv)); + if port_forwarding then begin + match Array.to_list Sys.argv with | [ _; "--wait"; "-t"; "nat"; "-I"; "DOCKER-INGRESS"; "-p"; proto; "--dport"; dport; "-j"; "DNAT"; "--to-destination"; ip_port ] -> let ip, port = parse_ip_port ip_port in insert { proto; dport; ip; port } @@ -90,5 +96,5 @@ let _ = delete { proto; dport; ip; port } | _ -> () - ); + end; Unix.execv _iptables Sys.argv From 17d110d770159f6e108adbd6829d2d0fb13e29b4 Mon Sep 17 00:00:00 2001 From: David Scott Date: Fri, 17 Jun 2016 17:25:57 +0100 Subject: [PATCH 10/10] iptables: log to /var/log/... rather than /var/run/log Signed-off-by: David Scott --- alpine/packages/iptables/main.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine/packages/iptables/main.ml b/alpine/packages/iptables/main.ml index 323f334af..aa9de21dc 100644 --- a/alpine/packages/iptables/main.ml +++ b/alpine/packages/iptables/main.ml @@ -16,7 +16,7 @@ type port = { port: string; (* container port *) } -let log_fd = Unix.openfile "/var/run/log/service-port-opener.log" [ Unix.O_WRONLY; Unix.O_APPEND; Unix.O_CREAT ] 0o0644 +let log_fd = Unix.openfile "/var/log/service-port-opener.log" [ Unix.O_WRONLY; Unix.O_APPEND; Unix.O_CREAT ] 0o0644 let logf fmt = Printf.ksprintf (fun s ->