diff --git a/balancer/xds/internal/regenerate_scripts/envoy-proto-gen.sh b/balancer/xds/internal/regenerate_scripts/envoy-proto-gen.sh new file mode 100755 index 00000000..19603f2b --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/envoy-proto-gen.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -ex +DATA_PLANE_API_VERSION=1935b52f94f7889ad9f538a17250e78cffd0af27 + +git clone git@github.com:envoyproxy/data-plane-api.git +git clone git@github.com:envoyproxy/protoc-gen-validate.git + +cd data-plane-api +git checkout $DATA_PLANE_API_VERSION +cp ../utils/WORKSPACE . +bazel clean --expunge + +# We download a local copy of the protoc-gen-validate repo to be used by bazel +# for customizing proto generated code import path. +# And we do a simple grep here to get the release version of the +# proto-gen-validate that gets used by data-plane-api. +PROTOC_GEN_VALIDATE=v$(grep "PGV_RELEASE =" ./bazel/repository_locations.bzl | sed -r 's/.*([0-9]+\.[0-9]+\.[0-9]+).*/\1/') + +cd ../protoc-gen-validate +git checkout $PROTOC_GEN_VALIDATE +git apply ../utils/protoc-gen-validate.patch + +cd ../data-plane-api + +# cleanup.sh remove all gogo proto related imports and labels. +../utils/cleanup.sh + +git apply ../utils/data-plane-api.patch +# proto-gen.sh build all packages required for grpc xds implementation and move +# proto generated code to grpc/balancer/xds/internal/proto subdirectory. +../utils/proto-gen.sh + diff --git a/balancer/xds/internal/regenerate_scripts/utils/README.md b/balancer/xds/internal/regenerate_scripts/utils/README.md new file mode 100644 index 00000000..96755154 --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/README.md @@ -0,0 +1,2 @@ +Run ./envoy-proto-gen.sh to generate xds proto generated code for grpc usage. +Make sure your $GOPATH is valid and you have grpc-go repo locally. \ No newline at end of file diff --git a/balancer/xds/internal/regenerate_scripts/utils/WORKSPACE b/balancer/xds/internal/regenerate_scripts/utils/WORKSPACE new file mode 100644 index 00000000..ed70a221 --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/WORKSPACE @@ -0,0 +1,57 @@ +workspace(name = "envoy_api") + +local_repository( + name = "com_lyft_protoc_gen_validate", + path = "../protoc-gen-validate", +) + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "io_bazel_rules_go", + urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.17.1/rules_go-0.17.1.tar.gz"], + sha256 = "6776d68ebb897625dead17ae510eac3d5f6342367327875210df44dbe2aeeb19", +) + +git_repository( + name = "com_github_golang_protobuf", + remote = "https://github.com/golang/protobuf", + commit = "aa810b61a9c79d51363740d207bb46cf8e620ed5", + shallow_since = "1534281267 -0700", + patches = [ + "@io_bazel_rules_go//third_party:com_github_golang_protobuf-gazelle.patch", + "@io_bazel_rules_go//third_party:com_github_golang_protobuf-extras.patch", + ], + patch_args = ["-p1"], +) + +load("//bazel:repositories.bzl", "api_dependencies") + +api_dependencies() + +load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains") +go_rules_dependencies() +go_register_toolchains() + +http_archive( + name = "bazel_gazelle", + urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"], + sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687", +) + +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") +gazelle_dependencies() + +bind( + name = "six", + actual = "@six_archive//:six", +) + +http_archive( + name = "six_archive", + sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", + build_file = "@com_google_protobuf//:six.BUILD", + url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55", +) + diff --git a/balancer/xds/internal/regenerate_scripts/utils/cleanup.sh b/balancer/xds/internal/regenerate_scripts/utils/cleanup.sh new file mode 100755 index 00000000..fd2893e6 --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +find . -name '*.proto' -print0 | +while IFS= read -r -d '' f +do + commands=( + # Import mangling. + -e 's#import "gogoproto/gogo.proto";##' + # Remove references to gogo.proto extensions. + -e 's#option (gogoproto\.[a-z_]\+) = \(true\|false\);##' + -e 's#\(, \)\?(gogoproto\.[a-z_]\+) = \(true\|false\),\?##' + # gogoproto removal can result in empty brackets. + -e 's# \[\]##' + # gogoproto removal can result in four spaces on a line by itself. + -e '/^ $/d' + ) + sed -i "${commands[@]}" "$f" + + # gogoproto removal can leave a comma on the last element in a list. + # This needs to run separately after all the commands above have finished + # since it is multi-line and rewrites the output of the above patterns. + sed -i -e '$!N; s#\(.*\),\([[:space:]]*\];\)#\1\2#; t; P; D;' "$f" +done diff --git a/balancer/xds/internal/regenerate_scripts/utils/data-plane-api.patch b/balancer/xds/internal/regenerate_scripts/utils/data-plane-api.patch new file mode 100644 index 00000000..b7256dfe --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/data-plane-api.patch @@ -0,0 +1,31 @@ +diff --git a/bazel/api_build_system.bzl b/bazel/api_build_system.bzl +index c68ccbd..e6cc8cb 100644 +--- a/bazel/api_build_system.bzl ++++ b/bazel/api_build_system.bzl +@@ -7,7 +7,7 @@ _PY_SUFFIX = "_py" + _CC_SUFFIX = "_cc" + _GO_PROTO_SUFFIX = "_go_proto" + _GO_GRPC_SUFFIX = "_go_grpc" +-_GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/" ++_GO_IMPORTPATH_PREFIX = "google.golang.org/grpc/balancer/xds/internal/proto/" + + def _Suffix(d, suffix): + return d + suffix +@@ -42,7 +42,7 @@ def api_py_proto_library(name, srcs = [], deps = [], has_services = 0): + def api_go_proto_library(name, proto, deps = []): + go_proto_library( + name = _Suffix(name, _GO_PROTO_SUFFIX), +- importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name), ++ importpath = _Suffix(_GO_IMPORTPATH_PREFIX + native.package_name() + "/", name), + proto = proto, + visibility = ["//visibility:public"], + deps = deps + [ +@@ -60,7 +60,7 @@ def api_go_proto_library(name, proto, deps = []): + def api_go_grpc_library(name, proto, deps = []): + go_grpc_library( + name = _Suffix(name, _GO_GRPC_SUFFIX), +- importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name), ++ importpath = _Suffix(_GO_IMPORTPATH_PREFIX + native.package_name() + "/", name), + proto = proto, + visibility = ["//visibility:public"], + deps = deps + [ diff --git a/balancer/xds/internal/regenerate_scripts/utils/proto-gen.sh b/balancer/xds/internal/regenerate_scripts/utils/proto-gen.sh new file mode 100755 index 00000000..b08fcc84 --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/proto-gen.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# packages is the collection of the packages that are required by xds for grpc. +packages=( + envoy/service/discovery/v2:ads_go_grpc + envoy/api/v2:eds_go_grpc + envoy/api/v2:cds_go_grpc + envoy/api/v2/core:address_go_proto + envoy/api/v2/core:base_go_proto + envoy/api/v2/endpoint:endpoint_go_proto + envoy/type:percent_go_proto + envoy/service/load_stats/v2:lrs_go_grpc + udpa/data/orca/v1:orca_load_report_go_proto + udpa/service/orca/v1:orca_go_grpc +) + +if [ -z $GOPATH ]; then echo 'empty $GOPATH, exiting.'; exit 1 +fi + +for i in ${packages[@]} +do + bazel build "$i" +done + +dest="$PWD/../../proto/" +rm -rf "$dest" +srcs=( + "find -L ./bazel-bin/envoy/ -name *.pb.go -print0" + "find -L ./bazel-bin/udpa/ -name *.pb.go -print0" + "find -L ./bazel-bin/ -name validate.pb.go -print0" +) + +for src in "${srcs[@]}" +do + eval "$src" | + while IFS= read -r -d '' origin + do + target="${origin##*proto/}" + final="$dest$target" + mkdir -p "${final%*/*}" + cp "$origin" "$dest$target" + done +done diff --git a/balancer/xds/internal/regenerate_scripts/utils/protoc-gen-validate.patch b/balancer/xds/internal/regenerate_scripts/utils/protoc-gen-validate.patch new file mode 100644 index 00000000..2e4b5c72 --- /dev/null +++ b/balancer/xds/internal/regenerate_scripts/utils/protoc-gen-validate.patch @@ -0,0 +1,26 @@ +diff --git a/validate/BUILD b/validate/BUILD +index af8c6c1..939d997 100644 +--- a/validate/BUILD ++++ b/validate/BUILD +@@ -31,7 +31,7 @@ py_proto_library( + + go_proto_library( + name = "go_default_library", +- importpath = "github.com/lyft/protoc-gen-validate/validate", ++ importpath = "google.golang.org/grpc/balancer/xds/internal/proto/validate", + proto = ":validate_proto", + visibility = ["//visibility:public"], + ) +diff --git a/validate/validate.proto b/validate/validate.proto +index 1c5e04a..7f5d4b0 100644 +--- a/validate/validate.proto ++++ b/validate/validate.proto +@@ -1,7 +1,7 @@ + syntax = "proto2"; + package validate; + +-option go_package = "github.com/lyft/protoc-gen-validate/validate"; ++option go_package = "google.golang.org/grpc/balancer/xds/internal/proto/validate"; + option java_package = "com.lyft.pgv.validate"; + + import "google/protobuf/descriptor.proto";