From 7eb24d6ff07b4731f08ee24b46c051097eaf066b Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 11 Aug 2023 19:46:44 -0700 Subject: [PATCH] Make custom tasks compatible with configuration cache Gradle is forcing a move away from using 'project' during task excution and because of some interactions there, this is easiest by making them real classes. That makes them start looking quite strange in the build file, so they are now moved to buildSrc/. We could have continued using Groovy, but it is weird in some ways that are more apparent when making classes and not just scripting. Instead, they were converted to Java. They are compatible with delayed configuration resolution as well. --- build.gradle | 103 +------------ buildSrc/build.gradle | 5 + .../io/grpc/gradle/CheckForUpdatesTask.java | 131 +++++++++++++++++ .../grpc/gradle/CheckPackageLeakageTask.java | 66 +++++++++ .../RequireUpperBoundDepsMatchTask.java | 137 ++++++++++++++++++ xds/build.gradle | 28 +--- 6 files changed, 349 insertions(+), 121 deletions(-) create mode 100644 buildSrc/build.gradle create mode 100644 buildSrc/src/main/java/io/grpc/gradle/CheckForUpdatesTask.java create mode 100644 buildSrc/src/main/java/io/grpc/gradle/CheckPackageLeakageTask.java create mode 100644 buildSrc/src/main/java/io/grpc/gradle/RequireUpperBoundDepsMatchTask.java diff --git a/build.gradle b/build.gradle index a3d741d154..26b7736f35 100644 --- a/build.gradle +++ b/build.gradle @@ -7,8 +7,9 @@ plugins { id 'com.google.cloud.tools.jib' apply false } +import io.grpc.gradle.CheckForUpdatesTask +import io.grpc.gradle.RequireUpperBoundDepsMatchTask import net.ltgt.gradle.errorprone.CheckSeverity -import org.gradle.util.GUtil subprojects { apply plugin: "checkstyle" @@ -275,15 +276,11 @@ subprojects { plugins.withId("java-library") { // Detect Maven Enforcer's dependencyConvergence failures. We only care // for artifacts used as libraries by others with Maven. - tasks.register('checkUpperBoundDeps') { - inputs.files(configurations.runtimeClasspath).withNormalizer(ClasspathNormalizer) - outputs.file("${buildDir}/tmp/${name}") // Fake output for UP-TO-DATE checking - doLast { - requireUpperBoundDepsMatch(configurations.runtimeClasspath, project) - } + tasks.register('checkUpperBoundDeps', RequireUpperBoundDepsMatchTask) { + configuration = configurations.getByName('runtimeClasspath') } tasks.named('assemble').configure { - dependsOn checkUpperBoundDeps + dependsOn tasks.named('checkUpperBoundDeps') } } @@ -456,64 +453,6 @@ subprojects { } } -class DepAndParents { - DependencyResult dep - List parents -} - -/** - * Make sure that Maven would select the same versions as Gradle selected. - * This is essentially the same as if we used Maven Enforcer's - * requireUpperBoundDeps for our artifacts. - */ -def requireUpperBoundDepsMatch(Configuration conf, Project project) { - // artifact name => version - Map golden = conf.resolvedConfiguration.resolvedArtifacts.collectEntries { - ResolvedArtifact it -> - ModuleVersionIdentifier id = it.moduleVersion.id - [id.group + ":" + id.name, id.version] - } - // Breadth-first search like Maven for dependency resolution - Queue queue = new ArrayDeque<>() - conf.incoming.resolutionResult.root.dependencies.each { - queue.add(new DepAndParents(dep: it, parents: [project.displayName])) - } - Set found = new HashSet<>() - while (!queue.isEmpty()) { - DepAndParents depAndParents = queue.remove() - ResolvedDependencyResult result = (ResolvedDependencyResult) depAndParents.dep - ModuleVersionIdentifier id = result.selected.moduleVersion - String artifact = id.group + ":" + id.name - if (found.contains(artifact)) - continue - found.add(artifact) - String version - if (result.requested instanceof ProjectComponentSelector) { - ProjectComponentSelector selector = (ProjectComponentSelector) result.requested - version = project.findProject(selector.projectPath).version - } else { - version = ((ModuleComponentSelector) result.requested).version - } - String goldenVersion = golden[artifact] - if (goldenVersion != version && "[$goldenVersion]" != version) { - throw new RuntimeException( - "Maven version skew: $artifact ($version != $goldenVersion) " - + "Bad version dependency path: " + depAndParents.parents - + " Run './gradlew $project.path:dependencies --configuration $conf.name' " - + "to diagnose") - } - result.selected.dependencies.each { - // Category.CATEGORY_ATTRIBUTE is the inappropriate Attribute because it is "desugared". - // https://github.com/gradle/gradle/issues/8854 - Attribute category = Attribute.of('org.gradle.category', String) - if (it.resolvedVariant.attributes.getAttribute(category) != Category.LIBRARY) - return - queue.add(new DepAndParents( - dep: it, parents: depAndParents.parents + [artifact + ":" + version])) - } - } -} - repositories { mavenCentral() google() @@ -546,34 +485,4 @@ configurations { } } -// Checks every dependency in the version catalog to see if there is a newer -// version available. The 'checkForUpdates' configuration restricts the -// versions considered. -tasks.register('checkForUpdates') { - doLast { - def updateConf = project.configurations.checkForUpdates - updateConf.setVisible(false) - updateConf.setTransitive(false) - def versionCatalog = project.extensions.getByType(VersionCatalogsExtension).named("libs") - versionCatalog.libraryAliases.each { name -> - def dep = versionCatalog.findLibrary(name).get().get() - - def oldConf = updateConf.copy() - def oldDep = project.dependencies.create( - group: dep.group, name: dep.name, version: dep.versionConstraint, classifier: 'pom') - oldConf.dependencies.add(oldDep) - def oldResolved = oldConf.resolvedConfiguration.resolvedArtifacts.iterator().next() - - def newConf = updateConf.copy() - def newDep = project.dependencies.create( - group: dep.group, name: dep.name, version: '+', classifier: 'pom') - newConf.dependencies.add(newDep) - def newResolved = newConf.resolvedConfiguration.resolvedArtifacts.iterator().next() - if (oldResolved != newResolved) { - def oldId = oldResolved.id.componentIdentifier - def newId = newResolved.id.componentIdentifier - println("libs.${name} = ${newId.group}:${newId.module} ${oldId.version} -> ${newId.version}") - } - } - } -} +tasks.register('checkForUpdates', CheckForUpdatesTask, project.configurations.checkForUpdates, "libs") diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 0000000000..3f79d30293 --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,5 @@ +tasks.withType(JavaCompile).configureEach { + it.options.compilerArgs += [ + "-Xlint:all", + ] +} diff --git a/buildSrc/src/main/java/io/grpc/gradle/CheckForUpdatesTask.java b/buildSrc/src/main/java/io/grpc/gradle/CheckForUpdatesTask.java new file mode 100644 index 0000000000..01f702c816 --- /dev/null +++ b/buildSrc/src/main/java/io/grpc/gradle/CheckForUpdatesTask.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.gradle; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import javax.inject.Inject; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.artifacts.VersionCatalog; +import org.gradle.api.artifacts.VersionCatalogsExtension; +import org.gradle.api.artifacts.result.ResolvedComponentResult; +import org.gradle.api.artifacts.result.ResolvedDependencyResult; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Nested; +import org.gradle.api.tasks.TaskAction; + +/** + * Checks every dependency in the version catalog to see if there is a newer version available. The + * passed configuration restricts the versions considered. + */ +public abstract class CheckForUpdatesTask extends DefaultTask { + private final Set libraries; + + @Inject + public CheckForUpdatesTask(Configuration updateConf, String catalog) { + updateConf.setVisible(false); + updateConf.setTransitive(false); + VersionCatalog versionCatalog = getProject().getExtensions().getByType(VersionCatalogsExtension.class).named(catalog); + Set libraries = new LinkedHashSet(); + for (String name : versionCatalog.getLibraryAliases()) { + org.gradle.api.artifacts.MinimalExternalModuleDependency dep = versionCatalog.findLibrary(name).get().get(); + + Configuration oldConf = updateConf.copy(); + Dependency oldDep = getProject().getDependencies().create( + depMap(dep.getGroup(), dep.getName(), dep.getVersionConstraint().toString(), "pom")); + oldConf.getDependencies().add(oldDep); + + Configuration newConf = updateConf.copy(); + Dependency newDep = getProject().getDependencies().create( + depMap(dep.getGroup(), dep.getName(), "+", "pom")); + newConf.getDependencies().add(newDep); + + libraries.add(new Library( + name, + oldConf.getIncoming().getResolutionResult().getRootComponent(), + newConf.getIncoming().getResolutionResult().getRootComponent())); + } + this.libraries = Collections.unmodifiableSet(libraries); + } + + private static Map depMap( + String group, String name, String version, String classifier) { + Map map = new HashMap<>(); + map.put("group", group); + map.put("name", name); + map.put("version", version); + map.put("classifier", classifier); + return map; + } + + @Nested + protected Set getLibraries() { + return libraries; + } + + @TaskAction + public void checkForUpdates() { + for (Library lib : libraries) { + String name = lib.getName(); + ModuleVersionIdentifier oldId = ((ResolvedDependencyResult) lib.getOldResult().get() + .getDependencies().iterator().next()).getSelected().getModuleVersion(); + ModuleVersionIdentifier newId = ((ResolvedDependencyResult) lib.getNewResult().get() + .getDependencies().iterator().next()).getSelected().getModuleVersion(); + if (oldId != newId) { + System.out.println(String.format( + "libs.%s = %s:%s %s -> %s", + name, newId.getGroup(), newId.getModule(), oldId.getVersion(), newId.getVersion())); + } + } + } + + public static final class Library { + private final String name; + private final Provider oldResult; + private final Provider newResult; + + public Library( + String name, Provider oldResult, + Provider newResult) { + this.name = name; + this.oldResult = oldResult; + this.newResult = newResult; + } + + @Input + public String getName() { + return name; + } + + @Input + public Provider getOldResult() { + return oldResult; + } + + @Input + public Provider getNewResult() { + return newResult; + } + } +} diff --git a/buildSrc/src/main/java/io/grpc/gradle/CheckPackageLeakageTask.java b/buildSrc/src/main/java/io/grpc/gradle/CheckPackageLeakageTask.java new file mode 100644 index 0000000000..1a3cf1fe0b --- /dev/null +++ b/buildSrc/src/main/java/io/grpc/gradle/CheckPackageLeakageTask.java @@ -0,0 +1,66 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.gradle; + +import java.io.File; +import java.io.IOException; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.TaskExecutionException; + +/** Verifies all class files within jar files are in a specified Java package. */ +public abstract class CheckPackageLeakageTask extends DefaultTask { + public CheckPackageLeakageTask() { + // Fake output for UP-TO-DATE checking + getOutputs().file(getProject().getLayout().getBuildDirectory().file("tmp/" + getName())); + } + + @Classpath + abstract ConfigurableFileCollection getFiles(); + + @Input + abstract Property getPrefix(); + + @TaskAction + public void checkLeakage() throws IOException { + String jarEntryPrefixName = getPrefix().get().replace('.', '/'); + boolean packageLeakDetected = false; + for (File jar : getFiles()) { + try (JarFile jf = new JarFile(jar)) { + for (Enumeration e = jf.entries(); e.hasMoreElements(); ) { + JarEntry entry = e.nextElement(); + if (entry.getName().endsWith(".class") + && !entry.getName().startsWith(jarEntryPrefixName)) { + packageLeakDetected = true; + System.out.println("WARNING: package leaked, may need relocation: " + entry.getName()); + } + } + } + } + if (packageLeakDetected) { + throw new TaskExecutionException(this, + new IllegalStateException("Resource leakage detected!")); + } + } +} diff --git a/buildSrc/src/main/java/io/grpc/gradle/RequireUpperBoundDepsMatchTask.java b/buildSrc/src/main/java/io/grpc/gradle/RequireUpperBoundDepsMatchTask.java new file mode 100644 index 0000000000..80d8f7bcb1 --- /dev/null +++ b/buildSrc/src/main/java/io/grpc/gradle/RequireUpperBoundDepsMatchTask.java @@ -0,0 +1,137 @@ +/* + * Copyright 2023 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.gradle; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.artifacts.component.ModuleComponentSelector; +import org.gradle.api.artifacts.component.ProjectComponentSelector; +import org.gradle.api.artifacts.result.DependencyResult; +import org.gradle.api.artifacts.result.ResolvedComponentResult; +import org.gradle.api.artifacts.result.ResolvedDependencyResult; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.attributes.Category; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.TaskAction; + +/** + * Verifies that Maven would select the same versions as Gradle selected. This is essentially the + * same as if we used Maven Enforcer's requireUpperBoundDeps in a Maven build. Gradle selects the + * upperBound, so if Maven selects a different version there is a downgrade. + */ +public abstract class RequireUpperBoundDepsMatchTask extends DefaultTask { + private final String projectPath; + private final String projectVersion; + + public RequireUpperBoundDepsMatchTask() { + projectPath = getProject().getPath(); + projectVersion = getProject().getVersion().toString(); + + // Fake output for UP-TO-DATE checking + getOutputs().file(getProject().getLayout().getBuildDirectory().file("tmp/" + getName())); + } + + public void setConfiguration(Configuration conf) { + getConfigurationName().set(conf.getName()); + getRoot().set(conf.getIncoming().getResolutionResult().getRootComponent()); + } + + @Input + abstract Property getConfigurationName(); + + @Input + abstract Property getRoot(); + + @TaskAction + public void checkDeps() { + // Category.CATEGORY_ATTRIBUTE is the inappropriate Attribute because it is "desugared". + // https://github.com/gradle/gradle/issues/8854 + Attribute category = Attribute.of("org.gradle.category", String.class); + + // Breadth-first search like Maven for dependency resolution. Check and Maven and Gradle would + // select the same version. + Queue queue = new ArrayDeque<>(); + for (DependencyResult dep : getRoot().get().getDependencies()) { + queue.add(new DepAndParents(dep, null, projectPath)); + } + Set found = new HashSet<>(); + while (!queue.isEmpty()) { + DepAndParents depAndParents = queue.remove(); + ResolvedDependencyResult result = (ResolvedDependencyResult) depAndParents.dep; + // Only libraries are "deps" in the typical sense, and non-libraries might not have versions. + if (result.getResolvedVariant().getAttributes().getAttribute(category) + != Category.LIBRARY) + return; + + ModuleVersionIdentifier id = result.getSelected().getModuleVersion(); + String artifact = id.getGroup() + ":" + id.getName(); + if (found.contains(artifact)) + continue; + found.add(artifact); + String mavenVersion; + if (result.getRequested() instanceof ProjectComponentSelector) { + // Assume all projects use the same version. + mavenVersion = projectVersion; + } else { + mavenVersion = ((ModuleComponentSelector) result.getRequested()).getVersion(); + } + String gradleVersion = id.getVersion(); + if (!mavenVersion.equals(gradleVersion) && !mavenVersion.equals("[" + gradleVersion + "]")) { + throw new RuntimeException(String.format( + "Maven version skew: %s (%s != %s) Bad version dependency path: %s" + + " Run './gradlew %s:dependencies --configuration %s' to diagnose", + artifact, mavenVersion, gradleVersion, depAndParents.getParents(), + projectPath, getConfigurationName().get())); + } + for (DependencyResult dep : result.getSelected().getDependencies()) { + queue.add(new DepAndParents(dep, depAndParents, artifact + ":" + mavenVersion)); + } + } + } + + static final class DepAndParents { + final DependencyResult dep; + final DepAndParents parent; + final String parentName; + + public DepAndParents(DependencyResult dep, DepAndParents parent, String parentName) { + this.dep = dep; + this.parent = parent; + this.parentName = parentName; + } + + public List getParents() { + List parents = new ArrayList<>(); + DepAndParents element = this; + while (element != null) { + parents.add(element.parentName); + element = element.parent; + } + Collections.reverse(parents); + return parents; + } + } +} diff --git a/xds/build.gradle b/xds/build.gradle index 7951fead5e..2f14dc6bf3 100644 --- a/xds/build.gradle +++ b/xds/build.gradle @@ -1,4 +1,4 @@ -import java.util.jar.JarFile +import io.grpc.gradle.CheckPackageLeakageTask; plugins { id "java" @@ -193,29 +193,9 @@ tasks.named("shadowJar").configure { exclude "**/*.proto" } -def checkPackageLeakage = tasks.register("checkPackageLeakage") { - inputs.files(shadowJar).withNormalizer(CompileClasspathNormalizer) - outputs.file("${buildDir}/tmp/${name}") // Fake output for UP-TO-DATE checking - doLast { - def jarEntryPrefixName = prefixName.replaceAll('\\.', '/') - shadowJar.outputs.getFiles().each { jar -> - def package_leak_detected = false - JarFile jf = new JarFile(jar) - jf.entries().each { entry -> - if (entry.name.endsWith(".class") && !entry.name.startsWith( - jarEntryPrefixName)) { - package_leak_detected = true - println "WARNING: package leaked, may need relocation: " + - entry.name - } - } - jf.close() - if (package_leak_detected) { - throw new TaskExecutionException(shadowJar, - new IllegalStateException("Resource leakage detected!")) - } - } - } +def checkPackageLeakage = tasks.register("checkPackageLeakage", CheckPackageLeakageTask) { + files.from tasks.named('shadowJar') + prefix = prefixName } tasks.named("test").configure {