From 6dae05f6ee72331b57b626a51c162fe0861ea349 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 20 Apr 2020 13:02:49 -0400 Subject: [PATCH 1/4] Add jCenter as a muzzle repository to catch things that are different from Maven Central For example in the latest release `jetty-server`'s version list was messed up: https://repo1.maven.org/maven2/org/glassfish/jersey/core/jersey-server/maven-metadata.xml but still available in jcenter: https://jcenter.bintray.com/org/glassfish/jersey/core/jersey-server/maven-metadata.xml --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index 71420ccb22..fa1fc21e77 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -37,8 +37,9 @@ class MuzzlePlugin implements Plugin { private static final AtomicReference TOOLING_LOADER = new AtomicReference<>() static { RemoteRepository central = new RemoteRepository.Builder("central", "default", "https://repo1.maven.org/maven2/").build() + RemoteRepository jcenter = new RemoteRepository.Builder("jcenter", "default", "https://jcenter.bintray.com/").build() RemoteRepository typesafe = new RemoteRepository.Builder("typesafe", "default", "https://repo.typesafe.com/typesafe/releases").build() - MUZZLE_REPOS = new ArrayList(Arrays.asList(central, typesafe)) + MUZZLE_REPOS = new ArrayList(Arrays.asList(central, jcenter, typesafe)) } @Override From 70f5ff7244c2dd85902119d5d0d50f258ac21f4c Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 20 Apr 2020 16:36:57 -0400 Subject: [PATCH 2/4] Add the ability to skip specific versions for muzzle This makes it much easier to bypass bad versions than splitting out the muzzle definition. --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 17 +++++++++-- .../commons-httpclient-2.gradle | 1 + .../elasticsearch/rest-6.4/rest-6.4.gradle | 9 +++--- .../transport-5.3/transport-5.3.gradle | 3 +- .../transport-5/transport-5.gradle | 3 +- .../transport-6/transport-6.gradle | 3 +- .../play-ws-2.1/play-ws-2.1.gradle | 28 ++++--------------- .../play-ws-2/play-ws-2.gradle | 28 ++++--------------- .../spring-webmvc-3.1.gradle | 28 ++++--------------- 9 files changed, 38 insertions(+), 82 deletions(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index fa1fc21e77..010069da56 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -206,7 +206,7 @@ class MuzzlePlugin implements Plugin { rangeRequest.setArtifact(directiveArtifact) final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest) - final List allVersionArtifacts = filterVersion(rangeResult.versions).collect { version -> + final List allVersionArtifacts = filterVersion(rangeResult.versions, muzzleDirective.skipVersions).collect { version -> new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", version.toString()) } @@ -237,7 +237,7 @@ class MuzzlePlugin implements Plugin { rangeRequest.setArtifact(directiveArtifact) final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest) - filterVersion(allRangeResult.versions).collect { version -> + filterVersion(allRangeResult.versions, muzzleDirective.skipVersions).collect { version -> if (!rangeResult.versions.contains(version)) { final MuzzleDirective inverseDirective = new MuzzleDirective() inverseDirective.group = muzzleDirective.group @@ -351,7 +351,7 @@ class MuzzlePlugin implements Plugin { /** * Filter out snapshot-type builds from versions list. */ - private static filterVersion(List list) { + private static filterVersion(List list, Set skipVersions) { list.removeIf { def version = it.toString().toLowerCase() return version.contains("rc") || @@ -363,6 +363,7 @@ class MuzzlePlugin implements Plugin { version.contains("-m") || version.contains("-dev") || version.contains("public_draft") || + skipVersions.contains(version) || version.matches(GIT_SHA_PATTERN) } return list @@ -388,6 +389,7 @@ class MuzzleDirective { String group String module String versions + Set skipVersions = new HashSet<>() List additionalDependencies = new ArrayList<>() boolean assertPass boolean assertInverse = false @@ -443,6 +445,7 @@ class MuzzleExtension { void pass(Action action) { final MuzzleDirective pass = objectFactory.newInstance(MuzzleDirective) action.execute(pass) + postConstruct(pass) pass.assertPass = true directives.add(pass) } @@ -450,7 +453,15 @@ class MuzzleExtension { void fail(Action action) { final MuzzleDirective fail = objectFactory.newInstance(MuzzleDirective) action.execute(fail) + postConstruct(fail) fail.assertPass = false directives.add(fail) } + + private postConstruct(MuzzleDirective directive) { + // Make skipVersions case insensitive. + directive.skipVersions = directive.skipVersions.collect { + it.toLowerCase() + } + } } diff --git a/dd-java-agent/instrumentation/commons-httpclient-2/commons-httpclient-2.gradle b/dd-java-agent/instrumentation/commons-httpclient-2/commons-httpclient-2.gradle index e938ab4157..443f2faf07 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2/commons-httpclient-2.gradle +++ b/dd-java-agent/instrumentation/commons-httpclient-2/commons-httpclient-2.gradle @@ -3,6 +3,7 @@ muzzle { group = "commons-httpclient" module = "commons-httpclient" versions = "[2.0,]" + skipVersions += "3.1-jenkins-1" // odd version in jcenter assertInverse = true } } diff --git a/dd-java-agent/instrumentation/elasticsearch/rest-6.4/rest-6.4.gradle b/dd-java-agent/instrumentation/elasticsearch/rest-6.4/rest-6.4.gradle index e2b4eb3d2c..0f67174137 100644 --- a/dd-java-agent/instrumentation/elasticsearch/rest-6.4/rest-6.4.gradle +++ b/dd-java-agent/instrumentation/elasticsearch/rest-6.4/rest-6.4.gradle @@ -45,9 +45,8 @@ dependencies { testCompile group: 'org.elasticsearch.plugin', name: 'transport-netty4-client', version: '6.4.0' // TODO: The tests are incompatible with 7.x. The instrumentation may be as well. - // FIXME: Lock to specific version due to bad deploy rollout. - latestDepTestCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: '6.8.3' - latestDepTestCompile group: 'org.elasticsearch.client', name: 'transport', version: '6.8.3' - latestDepTestCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '6.8.3' - latestDepTestCompile group: 'org.elasticsearch.plugin', name: 'transport-netty4-client', version: '6.8.3' + latestDepTestCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: '6.+' + latestDepTestCompile group: 'org.elasticsearch.client', name: 'transport', version: '6.+' + latestDepTestCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '6.+' + latestDepTestCompile group: 'org.elasticsearch.plugin', name: 'transport-netty4-client', version: '6.+' } diff --git a/dd-java-agent/instrumentation/elasticsearch/transport-5.3/transport-5.3.gradle b/dd-java-agent/instrumentation/elasticsearch/transport-5.3/transport-5.3.gradle index 57778b7c40..9a932653c4 100644 --- a/dd-java-agent/instrumentation/elasticsearch/transport-5.3/transport-5.3.gradle +++ b/dd-java-agent/instrumentation/elasticsearch/transport-5.3/transport-5.3.gradle @@ -8,8 +8,7 @@ muzzle { group = "org.elasticsearch.client" module = "transport" versions = "[5.3.0,6.0.0)" - // Work around for a bad release of 6.8.4 -// assertInverse = true + assertInverse = true } pass { group = "org.elasticsearch" diff --git a/dd-java-agent/instrumentation/elasticsearch/transport-5/transport-5.gradle b/dd-java-agent/instrumentation/elasticsearch/transport-5/transport-5.gradle index 715950ddd0..b1cd11ef13 100644 --- a/dd-java-agent/instrumentation/elasticsearch/transport-5/transport-5.gradle +++ b/dd-java-agent/instrumentation/elasticsearch/transport-5/transport-5.gradle @@ -8,8 +8,7 @@ muzzle { group = "org.elasticsearch.client" module = "transport" versions = "[5.0.0,5.3.0)" - // Work around for a bad release of 6.8.4 -// assertInverse = true + assertInverse = true } pass { group = "org.elasticsearch" diff --git a/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle b/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle index e8265c698a..9910b45735 100644 --- a/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle +++ b/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle @@ -8,8 +8,7 @@ muzzle { group = "org.elasticsearch.client" module = "transport" versions = "[6.0.0,6.8.4)" - // Work around for a bad release of 6.8.4 -// assertInverse = true + assertInverse = true } pass { group = "org.elasticsearch" diff --git a/dd-java-agent/instrumentation/play-ws-2.1/play-ws-2.1.gradle b/dd-java-agent/instrumentation/play-ws-2.1/play-ws-2.1.gradle index 7ac380a4ff..01b351ee04 100644 --- a/dd-java-agent/instrumentation/play-ws-2.1/play-ws-2.1.gradle +++ b/dd-java-agent/instrumentation/play-ws-2.1/play-ws-2.1.gradle @@ -14,45 +14,27 @@ testSets { } muzzle { - // 2.0.5 was a bad release fail { group = 'com.typesafe.play' module = 'play-ahc-ws-standalone_2.11' - versions = '[,2.0.4]' - } - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.11' - versions = '[2.0.6,)' + versions = '[,]' } - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.12' - versions = '[,2.0.4]' - } - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.12' - versions = '[2.0.6,2.1.0)' - } pass { group = 'com.typesafe.play' module = 'play-ahc-ws-standalone_2.12' versions = '[2.1.0,]' + skipVersions += '2.0.5' // Bad release + assertInverse = true } - // No Scala 2.13 versions below 2.0.6 exist - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.13' - versions = '[2.0.6,2.1.0)' - } pass { group = 'com.typesafe.play' module = 'play-ahc-ws-standalone_2.13' versions = '[2.1.0,]' + skipVersions += '2.0.5' // Bad release + assertInverse = true } } diff --git a/dd-java-agent/instrumentation/play-ws-2/play-ws-2.gradle b/dd-java-agent/instrumentation/play-ws-2/play-ws-2.gradle index 35b1638788..36c6036853 100644 --- a/dd-java-agent/instrumentation/play-ws-2/play-ws-2.gradle +++ b/dd-java-agent/instrumentation/play-ws-2/play-ws-2.gradle @@ -14,38 +14,20 @@ testSets { } muzzle { - // 2.0.5 was a bad release - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.11' - versions = '[,2.0.0)' - } pass { - group = 'com.typesafe.play' module = 'play-ahc-ws-standalone_2.11' - versions = '[2.0.0,2.0.4]' - } - pass { group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.11' - versions = '[2.0.6,]' + versions = '[2.0.0,]' + assertInverse = true } - fail { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.12' - versions = '[,2.0.0)' - } pass { group = 'com.typesafe.play' module = 'play-ahc-ws-standalone_2.12' - versions = '[2.0.0,2.0.4]' - } - pass { - group = 'com.typesafe.play' - module = 'play-ahc-ws-standalone_2.12' - versions = '[2.0.6,2.1.0)' + versions = '[2.0.0,2.1.0)' + skipVersions += '2.0.5' // Bad release + assertInverse = true } // No Scala 2.13 versions below 2.0.6 exist diff --git a/dd-java-agent/instrumentation/spring-webmvc-3.1/spring-webmvc-3.1.gradle b/dd-java-agent/instrumentation/spring-webmvc-3.1/spring-webmvc-3.1.gradle index b2c76e8668..7c79ca5f02 100644 --- a/dd-java-agent/instrumentation/spring-webmvc-3.1/spring-webmvc-3.1.gradle +++ b/dd-java-agent/instrumentation/spring-webmvc-3.1/spring-webmvc-3.1.gradle @@ -1,36 +1,20 @@ muzzle { - fail { - group = 'org.springframework' - module = 'spring-webmvc' - versions = "[,1.2.1)" - extraDependency "javax.servlet:javax.servlet-api:3.0.1" - } - // 1.2.1-1.2.4 have broken dependencies. - fail { - group = 'org.springframework' - module = 'spring-webmvc' - versions = "(1.2.4,3.1.0.RELEASE)" - extraDependency "javax.servlet:javax.servlet-api:3.0.1" - } pass { group = 'org.springframework' module = 'spring-webmvc' - versions = "[3.1.0.RELEASE,3.2.1.RELEASE)" - extraDependency "javax.servlet:javax.servlet-api:3.0.1" - } - // 3.2.1.RELEASE is missing a required class. (bad release?) - pass { - group = 'org.springframework' - module = 'spring-webmvc' - versions = "(3.2.1.RELEASE,]" + versions = "[3.1.0.RELEASE,]" + skipVersions += ['1.2.1', '1.2.2', '1.2.3', '1.2.4'] // broken releases... missing dependencies + skipVersions += '3.2.1.RELEASE' // missing a required class. (bad release?) extraDependency "javax.servlet:javax.servlet-api:3.0.1" + assertInverse = true } // FIXME: webmvc depends on web, so we need a separate integration for spring-web specifically. fail { group = 'org.springframework' module = 'spring-web' - versions = "(1.2.4,]" + versions = "[,]" + skipVersions += ['1.2.1', '1.2.2', '1.2.3', '1.2.4'] // broken releases... missing dependencies extraDependency "javax.servlet:javax.servlet-api:3.0.1" } } From 2743e64fd3d4ff4b6b0675a9dec7b939110da739 Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 20 Apr 2020 18:01:08 -0400 Subject: [PATCH 3/4] Exclude more weird versions. --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 12 +++++++++++- .../apache-httpclient-4/apache-httpclient-4.gradle | 1 + .../elasticsearch/transport-6/transport-6.gradle | 4 ++-- .../jax-rs-client-1.1/jax-rs-client-1.1.gradle | 4 +++- dd-java-agent/instrumentation/servlet/servlet.gradle | 2 ++ 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index 010069da56..22fc7197b5 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -37,9 +37,14 @@ class MuzzlePlugin implements Plugin { private static final AtomicReference TOOLING_LOADER = new AtomicReference<>() static { RemoteRepository central = new RemoteRepository.Builder("central", "default", "https://repo1.maven.org/maven2/").build() + RemoteRepository sonatype = new RemoteRepository.Builder("sonatype", "default", "https://oss.sonatype.org/content/repositories/releases/").build() RemoteRepository jcenter = new RemoteRepository.Builder("jcenter", "default", "https://jcenter.bintray.com/").build() + RemoteRepository spring = new RemoteRepository.Builder("spring", "default", "https://repo.spring.io/libs-release/").build() + RemoteRepository jboss = new RemoteRepository.Builder("jboss", "default", "https://repository.jboss.org/nexus/content/repositories/releases/").build() RemoteRepository typesafe = new RemoteRepository.Builder("typesafe", "default", "https://repo.typesafe.com/typesafe/releases").build() - MUZZLE_REPOS = new ArrayList(Arrays.asList(central, jcenter, typesafe)) + RemoteRepository akka = new RemoteRepository.Builder("akka", "default", "https://dl.bintray.com/akka/maven/").build() + RemoteRepository atlassian = new RemoteRepository.Builder("atlassian", "default", "https://maven.atlassian.com/content/repositories/atlassian-public/").build() + MUZZLE_REPOS = Arrays.asList(central, sonatype, jcenter, spring, jboss, typesafe, akka, atlassian) } @Override @@ -206,6 +211,9 @@ class MuzzlePlugin implements Plugin { rangeRequest.setArtifact(directiveArtifact) final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest) +// println "Range Request: " + rangeRequest +// println "Range Result: " + rangeResult + final List allVersionArtifacts = filterVersion(rangeResult.versions, muzzleDirective.skipVersions).collect { version -> new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", version.toString()) } @@ -362,6 +370,8 @@ class MuzzlePlugin implements Plugin { version.contains(".m") || version.contains("-m") || version.contains("-dev") || + version.contains("-ea") || + version.contains("-atlassian-") || version.contains("public_draft") || skipVersions.contains(version) || version.matches(GIT_SHA_PATTERN) diff --git a/dd-java-agent/instrumentation/apache-httpclient-4/apache-httpclient-4.gradle b/dd-java-agent/instrumentation/apache-httpclient-4/apache-httpclient-4.gradle index 985e98978d..c3e3feacc0 100644 --- a/dd-java-agent/instrumentation/apache-httpclient-4/apache-httpclient-4.gradle +++ b/dd-java-agent/instrumentation/apache-httpclient-4/apache-httpclient-4.gradle @@ -3,6 +3,7 @@ muzzle { group = "commons-httpclient" module = "commons-httpclient" versions = "[,4.0)" + skipVersions += '3.1-jenkins-1' } pass { group = "org.apache.httpcomponents" diff --git a/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle b/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle index 9910b45735..a080391992 100644 --- a/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle +++ b/dd-java-agent/instrumentation/elasticsearch/transport-6/transport-6.gradle @@ -7,13 +7,13 @@ muzzle { pass { group = "org.elasticsearch.client" module = "transport" - versions = "[6.0.0,6.8.4)" + versions = "[6.0.0,]" assertInverse = true } pass { group = "org.elasticsearch" module = "elasticsearch" - versions = "[6.0.0,)" + versions = "[6.0.0,]" assertInverse = true } } diff --git a/dd-java-agent/instrumentation/jax-rs-client-1.1/jax-rs-client-1.1.gradle b/dd-java-agent/instrumentation/jax-rs-client-1.1/jax-rs-client-1.1.gradle index 1b3f3414a6..1f5aeea9f4 100644 --- a/dd-java-agent/instrumentation/jax-rs-client-1.1/jax-rs-client-1.1.gradle +++ b/dd-java-agent/instrumentation/jax-rs-client-1.1/jax-rs-client-1.1.gradle @@ -2,7 +2,9 @@ muzzle { pass { group = "com.sun.jersey" module = "jersey-client" - versions = "[,]" + versions = "[1.1,]" + skipVersions += ['1.0.3-atlassian-1-logpatch', '1.8-atlassian-6'] + assertInverse = true } } diff --git a/dd-java-agent/instrumentation/servlet/servlet.gradle b/dd-java-agent/instrumentation/servlet/servlet.gradle index 50f9d1e4b1..0cca5e202b 100644 --- a/dd-java-agent/instrumentation/servlet/servlet.gradle +++ b/dd-java-agent/instrumentation/servlet/servlet.gradle @@ -9,6 +9,8 @@ muzzle { group = "javax.servlet" module = 'servlet-api' versions = "[,]" + skipVersions += '0' + assertInverse = true } } From 401ff4d4199f73e8e929d818864ae22b49f3cc1d Mon Sep 17 00:00:00 2001 From: Tyler Benson Date: Mon, 20 Apr 2020 23:42:05 -0400 Subject: [PATCH 4/4] Remove some repositories... Too many caused OOM in CI. --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index 22fc7197b5..2e8e56d45f 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -44,7 +44,8 @@ class MuzzlePlugin implements Plugin { RemoteRepository typesafe = new RemoteRepository.Builder("typesafe", "default", "https://repo.typesafe.com/typesafe/releases").build() RemoteRepository akka = new RemoteRepository.Builder("akka", "default", "https://dl.bintray.com/akka/maven/").build() RemoteRepository atlassian = new RemoteRepository.Builder("atlassian", "default", "https://maven.atlassian.com/content/repositories/atlassian-public/").build() - MUZZLE_REPOS = Arrays.asList(central, sonatype, jcenter, spring, jboss, typesafe, akka, atlassian) +// MUZZLE_REPOS = Arrays.asList(central, sonatype, jcenter, spring, jboss, typesafe, akka, atlassian) + MUZZLE_REPOS = Arrays.asList(central, jcenter, typesafe) } @Override