Agent: centrally manage proto files

In Kata 1.x, agent interface is defined in agent repo and vendored by runtime. But in Kata 2.0, agent and runtime will use ttrpc as protocol, and agent is using rust language, so runtime can't vendor agent again, have to compile from agent's protobuf files. This PR will hold proto files under src/agent, and compile it to rust/go sources for agent/runtime. Typing `make generate-protocols` under root of this repo or `src/agent` can see how to use it. Signed-off-by: bin liu <bin@hyper.sh>
2025-08-30 23:37:45 +00:00 · 2020-05-27 15:34:13 +08:00
parent 83f116b483
commit a3eb0c970e
7 changed files with 142 additions and 48 deletions
--- a/3
+++ b/3
@@ -18,3 +18,6 @@ test-agent:
 	make -C src/agent check

 test: test-runtime test-agent
+
+generate-protocols:
+	make -C src/agent generate-protocols
--- a/src/agent/Makefile
+++ b/src/agent/Makefile
@@ -25,6 +25,11 @@ export VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))

 BUILD_TYPE = release

+# set proto file to generate
+ifdef proto
+  PROTO_FILE=${proto}
+endif
+
 ARCH = $(shell uname -m)
 LIBC = musl
 TRIPLE = $(ARCH)-unknown-linux-$(LIBC)
@@ -122,3 +127,6 @@ help: show-summary
 	help \
 	show-header \
 	show-summary
+
+generate-protocols:
+	protocols/hack/update-generated-proto.sh "${PROTO_FILE}"
--- a/src/agent/protocols/hack/update-generated-proto.sh
+++ b/src/agent/protocols/hack/update-generated-proto.sh
@@ -1,68 +1,142 @@
 #!/bin/bash

+# //
+# // Copyright 2020 Ant Financial
+# //
+# // SPDX-License-Identifier: Apache-2.0
+# //
+
 die() {
-    echo $1
-    exit
+    cat <<EOT >&2
+====================================================================
+====                compile protocols failed                    ====
+
+$1
+
+====================================================================
+EOT
+    exit 1
 }

-get_source_version() {
-    if [ ! -d $GOPATH/src/$1 ]; then
-        go get -d -v $1
-    fi
-    [ $? -eq 0 ] || die "Failed to get $1"
-    if [ "$2" != "" ] ; then
-	    pushd "${GOPATH}/src/$1"
-        if [ $(git rev-parse HEAD) != $2 ] ; then
-            git checkout $2
-            [ $? -eq 0 ] || die "Failed to get $1 $2"
-        fi
-        popd
-    fi
+show_succeed_msg() {
+    echo "===================================================================="
+    echo "====                                                            ===="
+    echo "====                compile protocols succeed                   ===="
+    echo "====                                                            ===="
+    echo "===================================================================="
 }

-get_rs() {
-    local cmd="protoc --rust_out=./src/ --grpc_out=./src/,plugins=grpc:./src/ --plugin=protoc-gen-grpc=`which grpc_rust_plugin` -I ./protos/ ./protos/$1"
+show_usage() {
+    echo "===================================================================="
+    echo ""
+    echo "       USAGE: make PROTO_FILE=<xyz.proto> generate-protocols"
+    echo ""
+    echo "       Where PROTO_FILE may be:"
+    echo "         all: will compile all protocol buffer files"
+    echo ""
+    echo "       Or compile individually by using the exact proto file:"
+
+    # iterate over proto files
+    for file in "$@"
+    do
+        echo "         $file"
+    done
+
+    echo ""
+    echo "===================================================================="
+}
+
+generate_go_sources() {
+    local cmd="protoc -I$GOPATH/src/github.com/kata-containers/agent/vendor/github.com/gogo/protobuf:$GOPATH/src/github.com/kata-containers/agent/vendor:$GOPATH/src/github.com/gogo/protobuf:$GOPATH/src/github.com/gogo/googleapis:$GOPATH/src:/usr/local/include:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
+--gogottrpc_out=plugins=ttrpc+fieldpath,\
+import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
+\
+Mgithub.com/kata-containers/kata-containers/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
+\
+Mgithub.com/kata-containers/kata-containers/src/agent/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
+\
+Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto,Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/field_mask.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/wrappers.proto=github.com/gogo/protobuf/types,Mgoogle/rpc/status.proto=github.com/gogo/googleapis/google/rpc\
+:$GOPATH/src \
+$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos/$1"
+
    echo $cmd
    $cmd
-    [ $? -eq 0 ] || die "Failed to get rust from $1"
+    [ $? -eq 0 ] || die "Failed to generate golang file from $1"
 }

-if [ "$(basename $(pwd))" != "protocols" ] || [ ! -d "./hack/" ]; then
+generate_rust_sources() {
+    local cmd="protoc --rust_out=./protocols/src/ \
+--ttrpc_out=./protocols/src/,plugins=ttrpc:./protocols/src/ \
+--plugin=protoc-gen-ttrpc=`which ttrpc_rust_plugin` \
+-I $GOPATH/src/github.com/kata-containers/agent/vendor/github.com/gogo/protobuf:$GOPATH/src/github.com/kata-containers/agent/vendor:$GOPATH/src/github.com/gogo/protobuf:$GOPATH/src/github.com/gogo/googleapis:$GOPATH/src:/usr/local/include:$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos \
+$GOPATH/src/github.com/kata-containers/kata-containers/src/agent/protocols/protos/$1"
+
+    echo $cmd
+    $cmd
+    [ $? -eq 0 ] || die "Failed to generate rust file from $1"
+
+    if [ "$1" = "oci.proto" ]; then
+        # Need change Box<Self> to ::std::boxed::Box<Self> because there is another struct Box
+        sed 's/fn into_any(self: Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/g' ./protocols/src/oci.rs > ./protocols/src/new_oci.rs
+        sed 's/fn into_any(self: Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {/fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<dyn (::std::any::Any)> {/g' ./protocols/src/oci.rs > ./protocols/src/new_oci.rs
+        mv ./protocols/src/new_oci.rs ./protocols/src/oci.rs
+    fi;
+}
+
+if [ "$(basename $(pwd))" != "agent" ]; then
 	die "Please go to directory of protocols before execute this shell"
 fi
+
+# Protocol buffer files required to generate golang/rust bindings.
+proto_files_list=(agent.proto health.proto oci.proto github.com/kata-containers/agent/pkg/types/types.proto)
+
+if [ "$1" = "" ]; then
+    show_usage "${proto_files_list[@]}"
+    exit 1
+fi;
+
+# pre-requirement check
 which protoc
 [ $? -eq 0 ] || die "Please install protoc from github.com/protocolbuffers/protobuf"
+
 which protoc-gen-rust
 [ $? -eq 0 ] || die "Please install protobuf-codegen from github.com/pingcap/grpc-rs"
-which grpc_rust_plugin
-[ $? -eq 0 ] || die "Please install grpc_rust_plugin from github.com/pingcap/grpc-rs"

-if [ $UPDATE_PROTOS ]; then
-    if [ ! $GOPATH ]; then
-        die 'Need $GOPATH to get the proto files'
-    fi
+which ttrpc_rust_plugin
+[ $? -eq 0 ] || die "Please install ttrpc_rust_plugin from https://github.com/containerd/ttrpc-rust"

-    get_source_version "github.com/kata-containers/agent" ""
-    cp $GOPATH/src/github.com/kata-containers/agent/protocols/grpc/agent.proto ./protos/
-    cp $GOPATH/src/github.com/kata-containers/agent/protocols/grpc/oci.proto ./protos/
-    cp $GOPATH/src/github.com/kata-containers/agent/protocols/grpc/health.proto ./protos/
-    mkdir -p ./protos/github.com/kata-containers/agent/pkg/types/
-    cp $GOPATH/src/github.com/kata-containers/agent/pkg/types/types.proto ./protos/github.com/kata-containers/agent/pkg/types/
+which protoc-gen-gogottrpc
+[ $? -eq 0 ] || die "Please install protoc-gen-gogottrpc from https://github.com/containerd/ttrpc"

-    # The version is get from https://github.com/kata-containers/agent/blob/master/Gopkg.toml
-    get_source_version "github.com/gogo/protobuf" "4cbf7e384e768b4e01799441fdf2a706a5635ae7"
-    mkdir -p ./protos/github.com/gogo/protobuf/gogoproto/
-    cp $GOPATH/src/github.com/gogo/protobuf/gogoproto/gogo.proto ./protos/github.com/gogo/protobuf/gogoproto/
-    mkdir -p ./protos/google/protobuf/
-    cp $GOPATH/src/github.com/gogo/protobuf/protobuf/google/protobuf/empty.proto ./protos/google/protobuf/
-fi
+# do generate work
+target=$1

-get_rs agent.proto
-get_rs health.proto
-get_rs github.com/kata-containers/agent/pkg/types/types.proto
-get_rs google/protobuf/empty.proto
+# compile all proto files
+if [ "$target" = "all" ]; then
+    # compile all proto files
+    for f in ${proto_files_list[@]}; do
+        echo -e "\n   [golang] compiling ${f} ..."
+        generate_go_sources $f
+        echo -e "   [golang] ${f} compiled\n"

-get_rs oci.proto
-# Need change Box<Self> to ::std::boxed::Box<Self> because there is another struct Box
-sed 's/fn into_any(self: Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/fn into_any(self: ::std::boxed::Box<Self>) -> ::std::boxed::Box<::std::any::Any> {/g' src/oci.rs > src/new_oci.rs
-mv src/new_oci.rs src/oci.rs
+        echo -e "\n   [rust] compiling ${f} ..."
+        generate_rust_sources $f
+        echo -e "   [rust] ${f} compiled\n"
+    done
+else
+    # compile individual proto file
+    for f in ${proto_files_list[@]}; do
+        if [ "$target" = "$f" ]; then
+            echo -e "\n   [golang] compiling ${target} ..."
+            generate_go_sources $target
+            echo -e "   [golang] ${target} compiled\n"
+
+            echo -e "\n   [rust] compiling ${target} ..."
+            generate_rust_sources $target
+            echo -e "   [rust] ${target} compiled\n"
+        fi
+    done
+fi;
+
+# if have no errors, compilation will succeed
+show_succeed_msg
--- a/src/agent/protocols/protos/agent.proto
+++ b/src/agent/protocols/protos/agent.proto
@@ -7,10 +7,13 @@

 syntax = "proto3";

+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
 package grpc;

-import "oci.proto";
-import "github.com/kata-containers/agent/pkg/types/types.proto";
+import "github.com/kata-containers/kata-containers/src/agent/protocols/protos/oci.proto";
+import "github.com/kata-containers/kata-containers/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto";
+
 import "google/protobuf/empty.proto";

 // unstable
--- a/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto
+++ b/src/agent/protocols/protos/github.com/kata-containers/agent/pkg/types/types.proto
@@ -7,6 +7,8 @@

 syntax = "proto3";

+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols";
+
 package types;

 enum IPFamily {
--- a/src/agent/protocols/protos/health.proto
+++ b/src/agent/protocols/protos/health.proto
@@ -7,6 +7,8 @@

 syntax = "proto3";

+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
 package grpc;

 import "github.com/gogo/protobuf/gogoproto/gogo.proto";
--- a/src/agent/protocols/protos/oci.proto
+++ b/src/agent/protocols/protos/oci.proto
@@ -7,6 +7,8 @@

 syntax = "proto3";

+option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
+
 package grpc;

 import "github.com/gogo/protobuf/gogoproto/gogo.proto";