From 90777f4b397f5857a601e6e7641a0728b25d2699 Mon Sep 17 00:00:00 2001 From: Gary Date: Fri, 9 Nov 2018 17:09:24 -0500 Subject: [PATCH] Retire/remove version scan Remove version scan dead code and unnecessary classLoaderMatcher methods in instrumentations. Depends on #572 --- .../main/groovy/VersionScanExtension.groovy | 11 - .../src/main/groovy/VersionScanPlugin.groovy | 291 ------------------ ...earch53TransportClientInstrumentation.java | 6 - ...search5TransportClientInstrumentation.java | 6 - .../KafkaStreamsProcessorInstrumentation.java | 7 - .../lettuce-5/lettuce-5.gradle | 2 - 6 files changed, 323 deletions(-) delete mode 100644 buildSrc/src/main/groovy/VersionScanExtension.groovy delete mode 100644 buildSrc/src/main/groovy/VersionScanPlugin.groovy diff --git a/buildSrc/src/main/groovy/VersionScanExtension.groovy b/buildSrc/src/main/groovy/VersionScanExtension.groovy deleted file mode 100644 index 73136c0df6..0000000000 --- a/buildSrc/src/main/groovy/VersionScanExtension.groovy +++ /dev/null @@ -1,11 +0,0 @@ -class VersionScanExtension { - String group - String module - String versions - boolean scanMethods = false - boolean scanDependencies = false - String legacyGroup - String legacyModule - Map verifyPresent = Collections.emptyMap() - List verifyMissing = Collections.emptyList() -} diff --git a/buildSrc/src/main/groovy/VersionScanPlugin.groovy b/buildSrc/src/main/groovy/VersionScanPlugin.groovy deleted file mode 100644 index 3e00d76418..0000000000 --- a/buildSrc/src/main/groovy/VersionScanPlugin.groovy +++ /dev/null @@ -1,291 +0,0 @@ -import com.google.common.collect.Sets -import org.apache.maven.repository.internal.MavenRepositorySystemUtils -import org.eclipse.aether.DefaultRepositorySystemSession -import org.eclipse.aether.RepositorySystem -import org.eclipse.aether.RepositorySystemSession -import org.eclipse.aether.artifact.Artifact -import org.eclipse.aether.artifact.DefaultArtifact -import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory -import org.eclipse.aether.impl.DefaultServiceLocator -import org.eclipse.aether.repository.LocalRepository -import org.eclipse.aether.repository.RemoteRepository -import org.eclipse.aether.resolution.VersionRangeRequest -import org.eclipse.aether.resolution.VersionRangeResult -import org.eclipse.aether.spi.connector.RepositoryConnectorFactory -import org.eclipse.aether.spi.connector.transport.TransporterFactory -import org.eclipse.aether.transport.http.HttpTransporterFactory -import org.eclipse.aether.version.Version -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.objectweb.asm.ClassReader -import org.objectweb.asm.tree.ClassNode - -import java.util.concurrent.atomic.AtomicReference -import java.util.jar.JarEntry -import java.util.jar.JarFile - -/** - * Version syntax: https://maven.apache.org/enforcer/enforcer-rules/versionRanges.html - */ -class VersionScanPlugin implements Plugin { - - void apply(Project project) { - RepositorySystem system = newRepositorySystem() - RepositorySystemSession session = newRepositorySystemSession(system) - - project.extensions.create("versionScan", VersionScanExtension) - def scanVersions = project.task('scanVersions') { - group = 'VersionScan' - description = "Verify instrumentation against runtime classpaths." - } - - def hasRelevantTask = project.gradle.startParameter.taskNames.any { it.contains('scanVersions') } - hasRelevantTask |= project.gradle.startParameter.taskNames.any { it.contains('scanVersionsReport') } - hasRelevantTask |= project.gradle.startParameter.taskNames.any { it.contains('verifyVersionScan') } - - if (!hasRelevantTask) { - return - } - - if (!project.rootProject.version.toString().endsWith("-SNAPSHOT")) { - println "Skipping verifyVersionScan for release build." - project.task('verifyVersionScan').onlyIf { false } - return - } - -// println "Adding scan tasks for $project" - - Set allInclude = Sets.newConcurrentHashSet() - Set allExclude = Sets.newConcurrentHashSet() - AtomicReference> keyPresent = new AtomicReference(Collections.emptySet()) - AtomicReference> keyMissing = new AtomicReference(Collections.emptySet()) - - def scanVersionsReport = project.task('scanVersionsReport') { - description = "Prints the result of the scanVersions task" - doLast { - def inCommonPresent = keyPresent.get().size() - def inCommonMissing = keyMissing.get().size() - keyPresent.get().removeAll(allExclude) - keyMissing.get().removeAll(allInclude) - def exclusivePresent = keyPresent.get().size() - def exclusiveMissing = keyMissing.get().size() - - if (project.hasProperty("showClasses")) { - println "keyPresent: ${allInclude.size()}->$inCommonPresent->$exclusivePresent - ${keyPresent.get()}" - println "+++++++++++++++++++++" - println "keyMissing: ${allExclude.size()}->$inCommonMissing->$exclusiveMissing - ${keyMissing.get()}" - } else { - println "keyPresent: $inCommonPresent->$exclusivePresent" - println "keyMissing: $inCommonMissing->$exclusiveMissing" - } - } - } - if (project.gradle.startParameter.taskNames.any { it.contains('scanVersions') }) { - scanVersions.finalizedBy(scanVersionsReport) - } - - project.repositories { - mavenCentral() - jcenter() - } - - project.afterEvaluate { - String group = project.versionScan.group - String module = project.versionScan.module - String versions = project.versionScan.versions - Artifact artifact = new DefaultArtifact(group, module, "jar", versions) - Artifact allVersions = new DefaultArtifact(group, module, "jar", "(,)") - - String legacyGroup = project.versionScan.legacyGroup == null ? group : project.versionScan.legacyGroup - String legacyModule = project.versionScan.legacyModule == null ? module : project.versionScan.legacyModule - Artifact allLegacyVersions = new DefaultArtifact(legacyGroup, legacyModule, "jar", "(,)") - - VersionRangeRequest rangeRequest = new VersionRangeRequest() - rangeRequest.setRepositories(newRepositories(system, session)) - - rangeRequest.setArtifact(artifact) - VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest) - rangeRequest.setArtifact(allVersions) - VersionRangeResult allResult = system.resolveVersionRange(session, rangeRequest) - - def includeVersionSet = Sets.newHashSet(filter(rangeResult.versions).collect { - new DefaultArtifact(group, module, "jar", it.toString()) - }) - def excludeVersionSet = Sets.newHashSet(filter(allResult.versions).collect { - new DefaultArtifact(group, module, "jar", it.toString()) - }) - excludeVersionSet.removeAll(includeVersionSet) - - if (allVersions != allLegacyVersions) { -// println "Adding legacy versions $allLegacyVersions" - rangeRequest.setArtifact(allLegacyVersions) - VersionRangeResult allLegacyResult = system.resolveVersionRange(session, rangeRequest) - def legacyVersions = filter(allLegacyResult.versions) -// println "Found ${legacyVersions.size()} legacy versions for $legacyGroup:$legacyModule" - excludeVersionSet.addAll(legacyVersions.collect { - new DefaultArtifact(legacyGroup, legacyModule, "jar", it.toString()) - }) - } - - if (excludeVersionSet.empty) { - println "Found ${includeVersionSet.size()} versions, but none to exclude. Skipping..." - scanVersionsReport.enabled = false - return - } - - Map verifyPresent = project.versionScan.verifyPresent - List verifyMissing = project.versionScan.verifyMissing - - if (!verifyPresent.isEmpty() || !verifyMissing.isEmpty()) { - def verifyVersionScan = project.task('verifyVersionScan') { - description = "Validates that the configured classes and methods are only present where expected." - } - verifyVersionScan.doLast { - // This may already be done by the report task, but repeating for good measure. - keyPresent.get().removeAll(allExclude) - keyMissing.get().removeAll(allInclude) - - assert keyPresent.get() != [] || keyMissing.get() != [] - - def errors = [] - for (String className : verifyPresent.keySet()) { - if (project.versionScan.scanMethods && verifyPresent.get(className) == null) { - throw new AssertionError("When 'scanMethods' is enabled, a method must be configured for '$className'") - } else if (!project.versionScan.scanMethods && verifyPresent.get(className) != null) { - throw new AssertionError("When 'scanMethods' is not enabled, configured method must be null for '$className'") - } - - String identifier = project.versionScan.scanMethods ? "$className|${verifyPresent.get(className)}" : className - if (!keyPresent.get().contains(identifier)) { - errors << "not a 'keyPresent' identifier: $identifier" - } - } - for (String className : verifyMissing) { - if (!keyMissing.get().contains(className)) { - errors << "not a 'keyMissing' identifier: $className" - } - } - errors.each { - logger.error "Error for $group:$module - $it" - } - if (!errors.isEmpty()) { - throw new GradleException("Version scan verification failed.\n" + - "Errors listed above are likely the result of a new module " + - "being published to Maven, not a code change in this repo.") - } - } - - if (project.gradle.startParameter.taskNames.any { it.contains('scanVersions') }) { - scanVersions.finalizedBy(verifyVersionScan) - } - } - -// println "Scanning ${includeVersionSet.size()} included and ${excludeVersionSet.size()} excluded versions." -// println "Included: ${includeVersionSet.collect { it.version }}}" -// println "Excluded: ${excludeVersionSet.collect { it.version }}}" - - includeVersionSet.each { version -> - addScanTask("Include", new DefaultArtifact(version.groupId, version.artifactId, "jar", version.version), keyPresent, allInclude, project) - } - - excludeVersionSet.each { version -> - addScanTask("Exclude", new DefaultArtifact(version.groupId, version.artifactId, "jar", version.version), keyMissing, allExclude, project) - } - } - } - - def addScanTask(String label, Artifact artifact, AtomicReference> keyIdentifiers, Set allIdentifiers, Project project) { - def name = "scanVersion$label-$artifact.groupId-$artifact.artifactId-$artifact.version" - def config = project.configurations.create(name) - config.dependencies.add(project.dependencies.create("$artifact.groupId:$artifact.artifactId:$artifact.version") { - transitive = project.versionScan.scanDependencies - }) - - def task = project.task(name) { - doLast { - Set contentSet = Sets.newConcurrentHashSet() - project.configurations.getByName(name).resolvedConfiguration.files.each { jarFile -> - def jar = new JarFile(jarFile) - for (jarEntry in jar.entries()) { - if (jarEntry.name.endsWith(".class")) { - def className = jarEntry.name - className = className.replaceAll("/", ".") - className = className.replace(".class", "") - if (project.versionScan.scanMethods) { - findMethodNames(jar, jarEntry).each { - contentSet.add("$className|$it".toString()) - } - } else { - contentSet.add(className) - } - } - } - } - allIdentifiers.addAll(contentSet) - - if (!keyIdentifiers.compareAndSet(Collections.emptySet(), contentSet)) { - def intersection = Sets.intersection(keyIdentifiers.get(), contentSet) - keyIdentifiers.get().retainAll(intersection) - } - } - } - project.tasks.scanVersions.finalizedBy(task) - project.tasks.scanVersionsReport.dependsOn(task) - if (project.tasks.hasProperty("verifyVersionScan")) { - project.tasks.verifyVersionScan.dependsOn(task) - } - } - - def filter(List list) { - list.removeIf { - def version = it.toString().toLowerCase() - return version.contains("rc") || - version.contains(".cr") || - version.contains("alpha") || - version.contains("beta") || - version.contains("-b") || - version.contains(".m") || - version.contains("-dev") || - version.contains("public_draft") - } - return list - } - - def findMethodNames(JarFile jar, JarEntry entry) { - def stream = jar.getInputStream(entry) - - def classNode = new ClassNode() - def cr = new ClassReader(stream) - cr.accept(classNode, 0) - - return classNode.methods.collect { it.name } - } - - RepositorySystem newRepositorySystem() { - DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator() - locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class) - locator.addService(TransporterFactory.class, HttpTransporterFactory.class) - - return locator.getService(RepositorySystem.class) - } - - DefaultRepositorySystemSession newRepositorySystemSession(RepositorySystem system) { - DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession() - - def tempDir = File.createTempDir() - tempDir.deleteOnExit() - LocalRepository localRepo = new LocalRepository(tempDir) - session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo)) - - return session - } - - static List newRepositories(RepositorySystem system, RepositorySystemSession session) { - return new ArrayList(Arrays.asList(newCentralRepository())) - } - - private static RemoteRepository newCentralRepository() { - return new RemoteRepository.Builder("central", "default", "http://central.maven.org/maven2/").build() - } -} diff --git a/dd-java-agent/instrumentation/elasticsearch-transport-5.3/src/main/java/datadog/trace/instrumentation/elasticsearch5_3/Elasticsearch53TransportClientInstrumentation.java b/dd-java-agent/instrumentation/elasticsearch-transport-5.3/src/main/java/datadog/trace/instrumentation/elasticsearch5_3/Elasticsearch53TransportClientInstrumentation.java index 11ebc1b943..fa74e74d66 100644 --- a/dd-java-agent/instrumentation/elasticsearch-transport-5.3/src/main/java/datadog/trace/instrumentation/elasticsearch5_3/Elasticsearch53TransportClientInstrumentation.java +++ b/dd-java-agent/instrumentation/elasticsearch-transport-5.3/src/main/java/datadog/trace/instrumentation/elasticsearch5_3/Elasticsearch53TransportClientInstrumentation.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.elasticsearch5_3; -import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses; import static io.opentracing.log.Fields.ERROR_OBJECT; import static net.bytebuddy.matcher.ElementMatchers.isInterface; import static net.bytebuddy.matcher.ElementMatchers.isMethod; @@ -42,11 +41,6 @@ public class Elasticsearch53TransportClientInstrumentation extends Instrumenter. return not(isInterface()).and(named("org.elasticsearch.client.support.AbstractClient")); } - @Override - public ElementMatcher classLoaderMatcher() { - return classLoaderHasClasses("org.elasticsearch.percolator.TransportMultiPercolateAction"); - } - @Override public String[] helperClassNames() { return new String[] { diff --git a/dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5TransportClientInstrumentation.java b/dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5TransportClientInstrumentation.java index 1a1225e39a..5a99fb00e8 100644 --- a/dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5TransportClientInstrumentation.java +++ b/dd-java-agent/instrumentation/elasticsearch-transport-5/src/main/java/datadog/trace/instrumentation/elasticsearch5/Elasticsearch5TransportClientInstrumentation.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.elasticsearch5; -import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses; import static io.opentracing.log.Fields.ERROR_OBJECT; import static net.bytebuddy.matcher.ElementMatchers.isInterface; import static net.bytebuddy.matcher.ElementMatchers.isMethod; @@ -41,11 +40,6 @@ public class Elasticsearch5TransportClientInstrumentation extends Instrumenter.D return not(isInterface()).and(named("org.elasticsearch.client.support.AbstractClient")); } - @Override - public ElementMatcher classLoaderMatcher() { - return classLoaderHasClasses("org.elasticsearch.percolator.TransportMultiPercolateAction"); - } - @Override public String[] helperClassNames() { return new String[] { diff --git a/dd-java-agent/instrumentation/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsProcessorInstrumentation.java b/dd-java-agent/instrumentation/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsProcessorInstrumentation.java index 861b654145..a728bdd992 100644 --- a/dd-java-agent/instrumentation/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsProcessorInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsProcessorInstrumentation.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.kafka_streams; -import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses; import static io.opentracing.log.Fields.ERROR_OBJECT; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.isPackagePrivate; @@ -103,12 +102,6 @@ public class KafkaStreamsProcessorInstrumentation { return named("org.apache.kafka.streams.processor.internals.StreamTask"); } - @Override - public ElementMatcher classLoaderMatcher() { - return classLoaderHasClasses( - "org.apache.kafka.common.header.Header", "org.apache.kafka.common.header.Headers"); - } - @Override public String[] helperClassNames() { return HELPER_CLASS_NAMES; diff --git a/dd-java-agent/instrumentation/lettuce-5/lettuce-5.gradle b/dd-java-agent/instrumentation/lettuce-5/lettuce-5.gradle index 4c366a4f71..a9184228e7 100644 --- a/dd-java-agent/instrumentation/lettuce-5/lettuce-5.gradle +++ b/dd-java-agent/instrumentation/lettuce-5/lettuce-5.gradle @@ -4,8 +4,6 @@ ext { maxJavaVersionForTests = JavaVersion.VERSION_1_8 } -apply plugin: 'version-scan' - muzzle { pass { group = "io.lettuce"