From 3de7e74c5791d1fae8c5720958491fb97dd2e058 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 7 Jul 2022 07:26:38 -0700 Subject: [PATCH] xds: Build third-party protos in separate build step This dramatically shortens build time, even for full builds. A full assemble of xds on my laptop goes from 1m 46s to 33s at least because errorprone is disabled for the protos. --- build.gradle | 5 +- buildscripts/kokoro/gae-interop.sh | 1 + buildscripts/kokoro/windows32.bat | 1 + buildscripts/kokoro/windows64.bat | 1 + xds/build.gradle | 101 +++++++++++------- .../service/orca/v3/OpenRcaServiceGrpc.java | 0 .../api/v2/ClusterDiscoveryServiceGrpc.java | 0 .../api/v2/EndpointDiscoveryServiceGrpc.java | 0 .../api/v2/ListenerDiscoveryServiceGrpc.java | 0 .../api/v2/RouteDiscoveryServiceGrpc.java | 0 .../v2/ScopedRoutesDiscoveryServiceGrpc.java | 0 .../v2/VirtualHostDiscoveryServiceGrpc.java | 0 .../v2/AggregatedDiscoveryServiceGrpc.java | 0 .../v2/SecretDiscoveryServiceGrpc.java | 0 .../v3/AggregatedDiscoveryServiceGrpc.java | 0 .../v2/LoadReportingServiceGrpc.java | 0 .../v3/LoadReportingServiceGrpc.java | 0 .../v3/ClientStatusDiscoveryServiceGrpc.java | 0 18 files changed, 68 insertions(+), 41 deletions(-) rename xds/src/generated/{main => thirdparty}/grpc/com/github/xds/service/orca/v3/OpenRcaServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/ClusterDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/EndpointDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/ListenerDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/RouteDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/ScopedRoutesDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/api/v2/VirtualHostDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/discovery/v2/AggregatedDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/discovery/v2/SecretDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/load_stats/v2/LoadReportingServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/load_stats/v3/LoadReportingServiceGrpc.java (100%) rename xds/src/generated/{main => thirdparty}/grpc/io/envoyproxy/envoy/service/status/v3/ClientStatusDiscoveryServiceGrpc.java (100%) diff --git a/build.gradle b/build.gradle index cc4b30e8f2..3462c4350b 100644 --- a/build.gradle +++ b/build.gradle @@ -123,9 +123,8 @@ subprojects { release { java { srcDir "${generatedSourcePath}/release/grpc" } } } } else { - project.sourceSets { - main { java { srcDir "${generatedSourcePath}/main/grpc" } } - test { java { srcDir "${generatedSourcePath}/test/grpc" } } + project.sourceSets.each() { sourceSet -> + sourceSet.java { srcDir "${generatedSourcePath}/${sourceSet.name}/grpc" } } } } diff --git a/buildscripts/kokoro/gae-interop.sh b/buildscripts/kokoro/gae-interop.sh index b3973031d8..c4ce56cac5 100755 --- a/buildscripts/kokoro/gae-interop.sh +++ b/buildscripts/kokoro/gae-interop.sh @@ -29,6 +29,7 @@ cd "$GRPC_JAVA_DIR" ## Deploy the dummy 'default' version of the service ## GRADLE_FLAGS="--stacktrace -DgaeStopPreviousVersion=false -PskipCodegen=true -PskipAndroid=true" +export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" # Deploy the dummy 'default' version. We only require that it exists when cleanup() is called. # It ok if we race with another run and fail here, because the end result is idempotent. diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index 6d248d524f..7c6491cf97 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -29,6 +29,7 @@ SET FAIL_ON_WARNINGS=true SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% build" set GRADLEEXIT=%ERRORLEVEL% diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 0080c9a9c5..cf02336be3 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -28,6 +28,7 @@ SET FAIL_ON_WARNINGS=true SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true +SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" @rem make sure no daemons have any files open cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" diff --git a/xds/build.gradle b/xds/build.gradle index 9474675a13..8542d16b54 100644 --- a/xds/build.gradle +++ b/xds/build.gradle @@ -11,24 +11,41 @@ plugins { description = "gRPC: XDS plugin" -[compileJava].each() { - it.options.compilerArgs += [ - // valueOf(int) in RoutingPriority has been deprecated - "-Xlint:-deprecation", - // only has AutoValue annotation processor - "-Xlint:-processing", - ] - appendToProperty( - it.options.errorprone.excludedPaths, - ".*/build/generated/sources/annotationProcessor/java/.*", - "|") -} - evaluationDependsOn(project(':grpc-core').path) +sourceSets { + thirdparty { + java { + srcDir "${projectDir}/third_party/zero-allocation-hashing/main/java" + } + proto { + srcDir 'third_party/envoy/src/main/proto' + srcDir 'third_party/protoc-gen-validate/src/main/proto' + srcDir 'third_party/xds/src/main/proto' + srcDir 'third_party/googleapis/src/main/proto' + srcDir 'third_party/istio/src/main/proto' + } + } + test { + java { + srcDir "${projectDir}/third_party/zero-allocation-hashing/test/java" + } + } +} + +configurations { + pomDeps { + extendsFrom configurations.thirdpartyRuntimeClasspath, configurations.shadow + } +} + dependencies { - implementation project(':grpc-protobuf'), + thirdpartyCompileOnly libraries.javax.annotation + thirdpartyImplementation project(':grpc-protobuf'), project(':grpc-stub'), + libraries.opencensus.proto + implementation sourceSets.thirdparty.output + implementation project(':grpc-stub'), project(':grpc-core'), project(':grpc-services'), project(':grpc-auth'), @@ -36,7 +53,6 @@ dependencies { libraries.gson, libraries.re2j, libraries.auto.value.annotations, - libraries.opencensus.proto, libraries.protobuf.java.util def nettyDependency = implementation project(':grpc-netty') @@ -44,9 +60,8 @@ dependencies { testImplementation project(':grpc-core').sourceSets.test.output annotationProcessor libraries.auto.value - compileOnly libraries.javax.annotation, - // At runtime use the epoll included in grpc-netty-shaded - libraries.netty.transport.epoll + // At runtime use the epoll included in grpc-netty-shaded + compileOnly libraries.netty.transport.epoll testImplementation project(':grpc-testing'), project(':grpc-testing-proto') @@ -92,30 +107,36 @@ dependencies { } } -sourceSets { - main { - java { - srcDir "${projectDir}/third_party/zero-allocation-hashing/main/java" - } - proto { - srcDir 'third_party/envoy/src/main/proto' - srcDir 'third_party/protoc-gen-validate/src/main/proto' - srcDir 'third_party/xds/src/main/proto' - srcDir 'third_party/googleapis/src/main/proto' - srcDir 'third_party/istio/src/main/proto' - } - } - test { - java { - srcDir "${projectDir}/third_party/zero-allocation-hashing/test/java" - } - } +configureProtoCompilation() + +compileThirdpartyJava { + options.errorprone.enabled = false + options.compilerArgs += [ + // valueOf(int) in RoutingPriority has been deprecated + "-Xlint:-deprecation", + ] } -configureProtoCompilation() +checkstyleThirdparty { + enabled = false +} + +[compileJava].each() { + it.options.compilerArgs += [ + // TODO: remove + "-Xlint:-deprecation", + // only has AutoValue annotation processor + "-Xlint:-processing", + ] + appendToProperty( + it.options.errorprone.excludedPaths, + ".*/build/generated/sources/annotationProcessor/java/.*", + "|") +} jar { archiveClassifier = 'original' + from sourceSets.thirdparty.output } javadoc { @@ -211,7 +232,11 @@ publishing { pom.withXml { def dependenciesNode = new Node(null, 'dependencies') - project.configurations.shadow.allDependencies.each { dep -> + project.configurations.pomDeps.allDependencies.each { dep -> + if (dep.group == null && dep.name == 'unspecified') { + // Ignore the thirdparty self-dependency + return; + } def dependencyNode = dependenciesNode.appendNode('dependency') dependencyNode.appendNode('groupId', dep.group) dependencyNode.appendNode('artifactId', dep.name) diff --git a/xds/src/generated/main/grpc/com/github/xds/service/orca/v3/OpenRcaServiceGrpc.java b/xds/src/generated/thirdparty/grpc/com/github/xds/service/orca/v3/OpenRcaServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/com/github/xds/service/orca/v3/OpenRcaServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/com/github/xds/service/orca/v3/OpenRcaServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ClusterDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ClusterDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ClusterDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ClusterDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/EndpointDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/EndpointDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/EndpointDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/EndpointDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ListenerDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ListenerDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ListenerDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ListenerDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/RouteDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/RouteDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/RouteDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/RouteDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ScopedRoutesDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ScopedRoutesDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/ScopedRoutesDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/ScopedRoutesDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/VirtualHostDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/VirtualHostDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/api/v2/VirtualHostDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/api/v2/VirtualHostDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v2/AggregatedDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v2/AggregatedDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v2/AggregatedDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v2/AggregatedDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v2/SecretDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v2/SecretDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v2/SecretDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v2/SecretDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/discovery/v3/AggregatedDiscoveryServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/load_stats/v2/LoadReportingServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/load_stats/v2/LoadReportingServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/load_stats/v2/LoadReportingServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/load_stats/v2/LoadReportingServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/load_stats/v3/LoadReportingServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/load_stats/v3/LoadReportingServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/load_stats/v3/LoadReportingServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/load_stats/v3/LoadReportingServiceGrpc.java diff --git a/xds/src/generated/main/grpc/io/envoyproxy/envoy/service/status/v3/ClientStatusDiscoveryServiceGrpc.java b/xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/status/v3/ClientStatusDiscoveryServiceGrpc.java similarity index 100% rename from xds/src/generated/main/grpc/io/envoyproxy/envoy/service/status/v3/ClientStatusDiscoveryServiceGrpc.java rename to xds/src/generated/thirdparty/grpc/io/envoyproxy/envoy/service/status/v3/ClientStatusDiscoveryServiceGrpc.java