Merge pull request #1631 from samoht/multi-stage

miragesdk: simplify the build by using multi-stage Dockerfile
This commit is contained in:
Justin Cormack 2017-04-14 10:15:46 -05:00 committed by GitHub
commit 857d8b6448
15 changed files with 148 additions and 4681 deletions

View File

@ -3,9 +3,10 @@
.dev
obj/
hash
*.install
# Generated by jbuilder
dhcp-client/calf/.merlin
dhcp-client-calf/.merlin
dhcp-client/bpf/.merlin
dhcp-client/.merlin
sdk/.merlin
@ -14,7 +15,7 @@ test/.merlin
# Generated by `make dev`
_build/
main.native
dhcp-client/calf/dhcp_client
dhcp-client-calf/dhcp_client
src/bpf/.merlin
# Generated by the mirage tool

View File

@ -0,0 +1,88 @@
### Capnp compiler
FROM alpine:3.5 as capnp
RUN mkdir -p /src
RUN apk update && apk add autoconf automake libtool linux-headers git g++ make
RUN cd /src && git clone https://github.com/sandstorm-io/capnproto.git
WORKDIR /src/capnproto/c++
RUN ./setup-autotools.sh
RUN autoreconf -i
RUN ./configure
RUN make -j6 check
RUN make install
RUN which capnp
### SDK
FROM ocaml/opam:alpine-3.5_ocaml-4.04.0 as sdk
RUN git -C /home/opam/opam-repository pull && opam update -u
COPY --from=capnp /usr/local/bin/capnp /usr/local/bin/
COPY --from=capnp /usr/local/lib/libcapnpc-0.6-dev.so /usr/local/lib/
COPY --from=capnp /usr/local/lib/libcapnp-0.6-dev.so /usr/local/lib/
COPY --from=capnp /usr/local/lib/libkj-0.6-dev.so /usr/local/lib/
RUN sudo mkdir -p /src
USER opam
WORKDIR /src
RUN opam pin add jbuilder 1.0+beta7 -n
RUN opam depext -uiy cstruct lwt logs irmin-git rawlink tuntap astring rresult \
mirage-flow-lwt mirage-channel-lwt io-page decompress capnp
RUN opam list
COPY ./sdk /src/
COPY ./sdk.opam /src/
RUN sudo chown opam -R /src
RUN opam config exec -- jbuilder build -p sdk @install
RUN opam config exec -- jbuilder install sdk
### Privileged Container
FROM sdk as priv
RUN opam pin add tuntap 1.0.0 -n
RUN opam depext -iy bos cmdliner
RUN opam list
COPY ./dhcp-client /src/dhcp-client
RUN sudo chown opam -R /src
RUN opam config exec -- jbuilder build dhcp-client/main.exe
RUN sudo mkdir -p /bin
RUN sudo cp /src/_build/default/dhcp-client/main.exe /bin/dhcp-client
### Calf Container
FROM sdk as calf
RUN opam pin add charrua-client https://github.com/yomimono/charrua-client.git#state-halfway -n
RUN opam pin add mirage-net-fd 0.2.0 -n
RUN opam depext -iy mirage-net-fd charrua-client lwt mirage-types-lwt cmdliner
RUN opam list
COPY ./dhcp-client-calf/unikernel.ml /src/dhcp-client-calf/
COPY ./dhcp-client-calf/jbuild /src/dhcp-client-calf/
RUN sudo chown opam -R /src
RUN opam config exec -- jbuilder build dhcp-client-calf/unikernel.exe
RUN sudo mkdir -p /bin/calf
RUN sudo cp /src/_build/default/dhcp-client-calf/unikernel.exe /bin/calf/dhcp-client-calf
### Final build
FROM scratch
COPY --from=priv /bin /
COPY --from=calf /bin /
COPY dhcp-client-calf/config.json /calf/
CMD ["/dhcp-client", "-vv"]

View File

@ -1,37 +0,0 @@
FROM ocaml/opam:alpine-3.5_ocaml-4.04.0
RUN cd /home/opam/opam-repository && git pull && opam update -u
## pins for priv
# to bring eth0 up
RUN opam pin add tuntap 1.0.0 -n
RUN opam pin add mirage-net-fd 0.2.0 -n
## pins for calf
RUN opam pin add charrua-client https://github.com/yomimono/charrua-client.git#state-halfway -n
## depdendencies
RUN opam depext -iy \
irmin-unix cohttp decompress rawlink tuntap jbuilder irmin-watcher inotify \
rresult lwt capnp charrua-client mirage-net-fd ptime bos \
mirage-flow-lwt mirage-channel-lwt mirage-types-lwt
RUN sudo mkdir -p /src
COPY ./sdk /src/sdk
COPY ./dhcp-client /src/dhcp-client
RUN sudo chown opam -R /src
USER opam
WORKDIR /src
RUN opam list
RUN opam config exec -- jbuilder build dhcp-client/main.exe
RUN sudo cp /src/_build/default/dhcp-client/main.exe /dhcp-client
RUN apk add capnp
RUN opam config exec -- jbuilder build dhcp-client/calf/unikernel.exe
RUN sudo mkdir -p /calf
RUN sudo cp /src/_build/default/dhcp-client/calf/unikernel.exe /calf/dhcp-client-calf

View File

@ -1,28 +0,0 @@
FROM ocaml/opam:alpine-3.5_ocaml-4.04.0
RUN cd /home/opam/opam-repository && git pull && opam update -u
## pins for priv
# to bring eth0 up
RUN opam pin add tuntap 1.0.0 -n
RUN opam pin add mirage-net-fd 0.2.0 -n
## pins for calf
RUN opam pin add charrua-client https://github.com/yomimono/charrua-client.git#state-halfway -n
## depdendencies
RUN opam depext -iy \
irmin-unix cohttp decompress rawlink tuntap jbuilder irmin-watcher inotify \
rresult lwt capnp charrua-client mirage-net-fd ptime bos \
mirage-flow-lwt mirage-channel-lwt mirage-types-lwt
RUN sudo mkdir -p /src /bin
COPY . /src
COPY init-dev.sh /home/opam/init-dev.sh
USER opam
WORKDIR /src
ENTRYPOINT ["/bin/sh", "/home/opam/init-dev.sh"]

View File

@ -1,4 +0,0 @@
#FROM ocaml/opam:alpine-3.5_ocaml-4.04.0
FROM scratch
COPY obj ./
CMD ["/dhcp-client", "-vv"]

View File

@ -1,81 +1,39 @@
BASE=ocaml/opam:alpine-3.5_ocaml-4.04.0
FILES=$(shell find . -name jbuild) \
$(shell find sdk/ -name '*.ml') \
$(shell find sdk/ -name '*.mli') \
dhcp-client/bpf/dhcp.c dhcp-client/main.ml
IMAGE=dhcp-client
OBJS=obj/dhcp-client
dhcp-client/bpf/dhcp.c dhcp-client/main.ml \
dhcp-client-calf/unikernel.ml dhcp-client-calf/config.json
MIRAGE_COMPILE=mobylinux/mirage-compile:f903b0e1b4328271364cc63f123ac49d56739cef@sha256:a54d9ca84d3f5998dba92ce83d60d49289cee8908a8b0f6ec280d30ab8edf46c
CALF_OBJS=obj/calf/dhcp-client-calf
CALF_FILES=dhcp-client/calf/config.ml dhcp-client/calf/unikernel.ml \
dhcp-client/calf/jbuild
IMAGE=dhcp-client
default: push
@
.build: Dockerfile.build $(FILES)
docker build $(NO_CACHE) -t $(IMAGE):build -f Dockerfile.build -q . > .build || \
.build: Dockerfile $(FILES)
docker build $(NO_CACHE) -t $(IMAGE) -f Dockerfile .
docker build $(NO_CACHE) -t $(IMAGE) -f Dockerfile -q . > .build || \
(rm -f $@ && exit 1)
.pkg: Dockerfile.pkg $(OBJS) $(CALF_OBJS) obj/config.json
docker build --no-cache -t $(IMAGE):pkg -f Dockerfile.pkg -q . > .pkg || \
(rm -f $@ && exit 1)
.dev: Dockerfile.dev init-dev.sh
docker build $(NO_CACHE) -t $(IMAGE):dev -f Dockerfile.dev -q . > .dev || \
(rm -f $@ && exit 1)
enter-pkg: .pkg
docker run -v /bin/sh:/bin/sh -it --rm --entrypoint=/bin/sh $(shell cat .pkg)
enter-build: .build
docker run -it --rm $(shell cat .build)
enter-dev: .dev
docker run --privileged -it -v `pwd`:/src $(shell cat .dev)
#$(CALF_OBJS): $(CALF_FILES)
# mkdir -p obj/bin
# ( cd obj && \
# tar -C ../dhcp-client/calf -cf - $(CALF_FILES:dhcp-client/calf/%=%) | \
# docker run --rm -i --log-driver=none $(MIRAGE_COMPILE) -o dhcp-client-calf | \
# tar xf - || exit 1) && \
# touch $@
$(OBJS): .build $(FILES)
mkdir -p obj/calf
( cd obj && \
docker run --rm --net=none --log-driver=none -i $(IMAGE):build tar -cf - $(OBJS:obj/%=/%) | tar xf - ) && \
touch $@
$(CALF_OBJS): .build $(CALF_FILES)
mkdir -p obj/calf
( cd obj && \
docker run --rm --net=none --log-driver=none -i $(IMAGE):build tar -cf - $(CALF_OBJS:obj/%=/%) | tar xf - ) && \
touch $@
obj/config.json: dhcp-client/calf/config.json
mkdir -p obj/calf
cp $^ $@
hash: Makefile Dockerfile.build Dockerfile.pkg $(FILES) $(CALF_FILES) .build obj/config.json
hash: Makefile Dockerfile $(FILES) .build
{ cat $^; \
docker run --rm --entrypoint sh $(IMAGE):build -c 'cat /lib/apk/db/installed'; \
docker run --rm --entrypoint sh $(IMAGE):build -c 'opam list'; } \
docker run --rm --entrypoint sh $(IMAGE) -c 'cat /lib/apk/db/installed'; \
docker run --rm --entrypoint sh $(IMAGE) -c 'opam list'; } \
| sha1sum | sed 's/ .*//' > $@
push: hash .pkg
tag: .build
docker tag $(IMAGE) mobylinux/$(IMAGE):$(shell cat hash)
push: hash .build
docker pull $(BASE)
docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
(docker tag $(IMAGE):pkg mobylinux/$(IMAGE):$(shell cat hash) && \
(docker tag $(IMAGE) mobylinux/$(IMAGE):$(shell cat hash) && \
docker push mobylinux/$(IMAGE):$(shell cat hash))
clean::
rm -rf hash obj .build .pkg .dev
(docker rmi -f $(IMAGE):build || echo ok)
(docker rmi -f $(IMAGE):pkg || echo ok)
(docker rmi -f $(IMAGE):dev || echo ok)
rm -rf hash .build
(docker rmi -f $(IMAGE) || echo ok)
#### DEV
@ -85,11 +43,10 @@ test:
jbuilder runtest --dev
dev-clean:
rm -rf _build dhcp-client/calf/_build
rm -rf _build
dev:
jbuilder build dhcp-client/main.exe --dev
jbuilder build dhcp-client/calf/unikernel.exe --dev
# cd dhcp-client/calf && mirage configure && make
jbuilder build dhcp-client-calf/unikernel.exe --dev
.DELETE_ON_ERROR:

View File

@ -3,6 +3,6 @@
(executables
((names (unikernel))
(libraries (sdk mirage-net-fd lwt charrua-client.mirage charrua-client
lwt.unix))
lwt.unix cmdliner fmt.cli logs.fmt logs.cli fmt.tty))
(flags (-cclib -static))
))

View File

@ -1,57 +0,0 @@
open Mirage
(* create a new device for mirage-net-fd *)
(* FIXME: should check it is invoked only with the unix backend *)
(* FIXME: this is a temporary solution, this should be exposed
as a ukvm/virtio device *)
let netif_of_fd id = impl @@
let key = Key.abstract id in
object
inherit base_configurable
method ty = network
val name = Functoria_app.Name.create "net" ~prefix:"net"
method name = name
method module_name = "Netif_fd"
method keys = [ key ]
method packages = Key.pure [ package "mirage-net-fd" ]
method connect _ modname _ =
Fmt.strf "@[let (key: int) = %a in@,
%s.connect (Obj.magic key: Unix.file_descr)@]"
Key.serialize_call key modname
method configure i =
Ok ()
end
let dhcp_codes =
let doc = Key.Arg.info ~docv:"OPT" ~doc:"DHCP options." ["c";"codes"] in
Key.(abstract @@ create "codes" Arg.(opt (list string) [] doc))
let net =
let doc =
Key.Arg.info ~docv:"FD" ~doc:"Network interface" ["net"]
in
let key = Key.(create "input" Arg.(opt int 3 doc)) in
netif_of_fd key
let ctl =
let doc =
Key.Arg.info ~docv:"FD" ~doc:"Control interface" ["ctl"]
in
let key = Key.(create "output" Arg.(opt int 4 doc)) in
netif_of_fd key
let keys = [dhcp_codes]
let packages = [
package "jsonm";
package "charrua-client";
package "duration";
package "charrua-client" ~sublibs:["mirage"];
package "cohttp" ~sublibs:["lwt"]
]
let main =
foreign ~keys ~packages "Unikernel.Main"
(time @-> network @-> network @-> job)
let () = register "dhcp-client" [main $ default_time $ net $ ctl]

View File

@ -2,6 +2,6 @@
(executables
((names (main))
(libraries (sdk bpf_dhcp bos))
(libraries (sdk bpf_dhcp bos cmdliner fmt.cli logs.fmt logs.cli fmt.tty))
(flags (-cclib -static))
))

View File

@ -0,0 +1,24 @@
opam-version: "1.2"
name: "sdk"
version: "dev"
maintainer: "Thomas Gazagnaire <thomas@gazagnaire.org>"
authors: "Thomas Gazagnaire <thomas@gazagnaire.org>"
homepage: "https://github.com/linuxkit/linuxkit"
bug-reports: "https://github.com/linuxkit/linuxkit/issues"
license: "Apache"
dev-repo: "https://github.com/linuxkit/linuxkit.git"
build: ["jbuilder" "build" "-p" name "-j" jobs "@install"]
depends: [
"jbuilder" {build & >= "1.0+beta7"}
"ocamlfind" {build}
"cstruct"
"lwt"
"logs" "astring" "rresult"
"mirage-flow-lwt"
"mirage-channel-lwt"
"io-page"
"irmin-git"
"decompress"
"capnp"
]

View File

@ -2,17 +2,17 @@
(library
((name sdk)
(libraries (threads cstruct.lwt cmdliner fmt.cli logs.fmt logs.cli fmt.tty
decompress irmin irmin-git lwt.unix rawlink tuntap dispatch
irmin-watcher inotify astring rresult mirage-flow-lwt
mirage-channel-lwt io-page.unix ipaddr capnp))
(public_name sdk)
(libraries (cstruct.lwt decompress irmin irmin-git lwt.unix rawlink
tuntap astring rresult mirage-flow-lwt capnp
mirage-channel-lwt io-page.unix ipaddr))
))
;(rule
; ((targets (proto.ml proto.mli))
; (deps (proto.capnp))
; (action (progn
; (run capnp compile -o ocaml ${<})
; (system "mv proto.ml proto.ml.in")
; (system "echo '[@@@ocaml.warning \"-A\"]\n' > proto.ml")
; (system "cat proto.ml.in >> proto.ml")))))
(rule
((targets (proto.ml proto.mli))
(deps (proto.capnp))
(action (progn
(run capnp compile -o ocaml ${<})
(system "mv proto.ml proto.ml.in")
(system "echo '[@@@ocaml.warning \"-A\"]\n' > proto.ml")
(system "cat proto.ml.in >> proto.ml")))))

File diff suppressed because it is too large Load Diff

View File

@ -1,109 +0,0 @@
type ro = Capnp.Message.ro
type rw = Capnp.Message.rw
module type S = sig
type 'cap message_t
type reader_t_Request_14112192289179464829
type builder_t_Request_14112192289179464829
type reader_t_Response_16897334327181152309
type builder_t_Response_16897334327181152309
module Reader : sig
type array_t
type builder_array_t
type pointer_t
module Response : sig
type t = reader_t_Response_16897334327181152309
type builder_t = builder_t_Response_16897334327181152309
type unnamed_union_t =
| Ok of string
| Error of string
| Undefined of int
val get : t -> unnamed_union_t
val id_get : t -> int32
val id_get_int_exn : t -> int
val of_message : 'cap message_t -> t
val of_builder : builder_t -> t
end
module Request : sig
type t = reader_t_Request_14112192289179464829
type builder_t = builder_t_Request_14112192289179464829
type unnamed_union_t =
| Write of string
| Read
| Delete
| Undefined of int
val get : t -> unnamed_union_t
val id_get : t -> int32
val id_get_int_exn : t -> int
val has_path : t -> bool
val path_get : t -> (ro, string, array_t) Capnp.Array.t
val path_get_list : t -> string list
val path_get_array : t -> string array
val of_message : 'cap message_t -> t
val of_builder : builder_t -> t
end
end
module Builder : sig
type array_t = Reader.builder_array_t
type reader_array_t = Reader.array_t
type pointer_t
module Response : sig
type t = builder_t_Response_16897334327181152309
type reader_t = reader_t_Response_16897334327181152309
type unnamed_union_t =
| Ok of string
| Error of string
| Undefined of int
val get : t -> unnamed_union_t
val ok_set : t -> string -> unit
val error_set : t -> string -> unit
val id_get : t -> int32
val id_get_int_exn : t -> int
val id_set : t -> int32 -> unit
val id_set_int_exn : t -> int -> unit
val of_message : rw message_t -> t
val to_message : t -> rw message_t
val to_reader : t -> reader_t
val init_root : ?message_size:int -> unit -> t
end
module Request : sig
type t = builder_t_Request_14112192289179464829
type reader_t = reader_t_Request_14112192289179464829
type unnamed_union_t =
| Write of string
| Read
| Delete
| Undefined of int
val get : t -> unnamed_union_t
val write_set : t -> string -> unit
val read_set : t -> unit
val delete_set : t -> unit
val id_get : t -> int32
val id_get_int_exn : t -> int
val id_set : t -> int32 -> unit
val id_set_int_exn : t -> int -> unit
val has_path : t -> bool
val path_get : t -> (rw, string, array_t) Capnp.Array.t
val path_get_list : t -> string list
val path_get_array : t -> string array
val path_set : t -> (rw, string, array_t) Capnp.Array.t -> (rw, string, array_t) Capnp.Array.t
val path_set_list : t -> string list -> (rw, string, array_t) Capnp.Array.t
val path_set_array : t -> string array -> (rw, string, array_t) Capnp.Array.t
val path_init : t -> int -> (rw, string, array_t) Capnp.Array.t
val of_message : rw message_t -> t
val to_message : t -> rw message_t
val to_reader : t -> reader_t
val init_root : ?message_size:int -> unit -> t
end
end
end
module Make (MessageWrapper : Capnp.MessageSig.S) :
(S with type 'cap message_t = 'cap MessageWrapper.Message.t
and type Reader.pointer_t = ro MessageWrapper.Slice.t option
and type Builder.pointer_t = rw MessageWrapper.Slice.t
)