Merge pull request #452 from DataDog/ark/version-scan-muzzle

Add version scanning to muzzle gradle plugin
This commit is contained in:
Andrew Kent 2018-08-24 11:10:52 -07:00 committed by GitHub
commit f833218f44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 373 additions and 106 deletions

View File

@ -1,4 +0,0 @@
class MuzzleExtension {
String group
String module
}

View File

@ -1,32 +1,61 @@
import org.apache.maven.repository.internal.MavenRepositorySystemUtils
import org.eclipse.aether.DefaultRepositorySystemSession
import org.eclipse.aether.RepositorySystem
import org.eclipse.aether.RepositorySystemSession
import org.eclipse.aether.artifact.Artifact
import org.eclipse.aether.artifact.DefaultArtifact
import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory
import org.eclipse.aether.impl.DefaultServiceLocator
import org.eclipse.aether.repository.LocalRepository
import org.eclipse.aether.repository.RemoteRepository
import org.eclipse.aether.resolution.VersionRangeRequest
import org.eclipse.aether.resolution.VersionRangeResult
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory
import org.eclipse.aether.spi.connector.transport.TransporterFactory
import org.eclipse.aether.transport.http.HttpTransporterFactory
import org.eclipse.aether.version.Version
import org.gradle.api.Action
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.model.ObjectFactory
import java.lang.reflect.Method
/**
* muzzle task plugin which runs muzzle validation against an instrumentation's compile-time dependencies.
*
* <p/>TODO: merge this with version scan
* muzzle task plugin which runs muzzle validation against a range of dependencies.
*/
class MuzzlePlugin implements Plugin<Project> {
/**
* Remote repositories used to query version ranges and fetch dependencies
*/
private static final List<RemoteRepository> MUZZLE_REPOS
static {
RemoteRepository central = new RemoteRepository.Builder("central", "default", "http://central.maven.org/maven2/").build()
MUZZLE_REPOS = new ArrayList<RemoteRepository>(Arrays.asList(central))
}
@Override
void apply(Project project) {
def bootstrapProject = project.rootProject.getChildProjects().get('dd-java-agent').getChildProjects().get('agent-bootstrap')
def toolingProject = project.rootProject.getChildProjects().get('dd-java-agent').getChildProjects().get('agent-tooling')
project.extensions.create("muzzle", MuzzleExtension)
def compileMuzzle = project.task('compileMuzzle') {
// not adding user and group to hide this from `gradle tasks`
}
project.extensions.create("muzzle", MuzzleExtension, project.objects)
// compileMuzzle compiles all projects required to run muzzle validation.
// Not adding group and description to keep this task from showing in `gradle tasks`.
def compileMuzzle = project.task('compileMuzzle')
def muzzle = project.task('muzzle') {
group = 'Muzzle'
description = "Run instrumentation muzzle on compile time dependencies"
doLast {
final ClassLoader userCL = createUserClassLoader(project, bootstrapProject)
if (!project.muzzle.directives.any { it.assertPass }) {
project.getLogger().info('No muzzle pass directives configured. Asserting pass against instrumentation compile-time dependencies')
final ClassLoader userCL = createCompileDepsClassLoader(project, bootstrapProject)
final ClassLoader agentCL = createDDClassloader(project, toolingProject)
// find all instrumenters, get muzzle, and assert
Method assertionMethod = agentCL.loadClass('datadog.trace.agent.tooling.muzzle.MuzzleVersionScanPlugin')
.getMethod('assertInstrumentationNotMuzzled', ClassLoader.class)
assertionMethod.invoke(null, userCL)
.getMethod('assertInstrumentationMuzzled', ClassLoader.class, boolean.class)
assertionMethod.invoke(null, userCL, true)
}
}
}
def printReferences = project.task('printReferences') {
@ -43,19 +72,52 @@ class MuzzlePlugin implements Plugin<Project> {
project.tasks.compileMuzzle.dependsOn(toolingProject.tasks.compileJava)
project.afterEvaluate {
project.tasks.compileMuzzle.dependsOn(project.tasks.compileJava)
if (project.tasks.getNames().contains("compileScala")) {
if (project.tasks.getNames().contains('compileScala')) {
project.tasks.compileMuzzle.dependsOn(project.tasks.compileScala)
}
}
project.tasks.muzzle.dependsOn(project.tasks.compileMuzzle)
project.tasks.printReferences.dependsOn(project.tasks.compileMuzzle)
def hasRelevantTask = project.gradle.startParameter.taskNames.any { taskName ->
// removing leading ':' if present
taskName = taskName.replaceFirst('^:', '')
String muzzleTaskPath = project.path.replaceFirst('^:', '')
return 'muzzle' == taskName || "${muzzleTaskPath}:muzzle" == taskName
}
if (!hasRelevantTask) {
// Adding muzzle dependencies has a large config overhead. Stop unless muzzle is explicitly run.
return
}
final RepositorySystem system = newRepositorySystem()
final RepositorySystemSession session = newRepositorySystemSession(system)
project.afterEvaluate {
// use runAfter to set up task finalizers in version order
Task runAfter = project.tasks.muzzle
for (MuzzleDirective muzzleDirective : project.muzzle.directives) {
project.getLogger().info("configured ${muzzleDirective.assertPass ? 'pass' : 'fail'} directive: ${muzzleDirective.group}:${muzzleDirective.module}:${muzzleDirective.versions}")
muzzleDirectiveToArtifacts(muzzleDirective, system, session).collect() { Artifact singleVersion ->
runAfter = addMuzzleTask(muzzleDirective, singleVersion, project, runAfter, bootstrapProject, toolingProject)
}
if (muzzleDirective.assertInverse) {
inverseOf(muzzleDirective, system, session).collect() { MuzzleDirective inverseDirective ->
muzzleDirectiveToArtifacts(inverseDirective, system, session).collect() { Artifact singleVersion ->
runAfter = addMuzzleTask(inverseDirective, singleVersion, project, runAfter, bootstrapProject, toolingProject)
}
}
}
}
}
}
/**
* Create a classloader with core agent classes and project instrumentation on the classpath.
*/
private ClassLoader createDDClassloader(Project project, Project toolingProject) {
private static ClassLoader createDDClassloader(Project project, Project toolingProject) {
project.getLogger().info("Creating dd classpath for: " + project.getName())
Set<URL> ddUrls = new HashSet<>()
for (File f : toolingProject.sourceSets.main.runtimeClasspath.getFiles()) {
@ -71,11 +133,11 @@ class MuzzlePlugin implements Plugin<Project> {
}
/**
* Create a classloader with user/library classes on the classpath.
* Create a classloader with all compile-time dependencies on the classpath
*/
private ClassLoader createUserClassLoader(Project project, Project bootstrapProject) {
private static ClassLoader createCompileDepsClassLoader(Project project, Project bootstrapProject) {
List<URL> userUrls = new ArrayList<>()
project.getLogger().info("Creating user classpath for: " + project.getName())
project.getLogger().info("Creating compile-time classpath for: " + project.getName())
for (File f : project.configurations.compileOnly.getFiles()) {
project.getLogger().info('--' + f)
userUrls.add(f.toURI().toURL())
@ -86,4 +148,201 @@ class MuzzlePlugin implements Plugin<Project> {
}
return new URLClassLoader(userUrls.toArray(new URL[0]), (ClassLoader) null)
}
/**
* Create a classloader with dependencies for a single muzzle task.
*/
private static ClassLoader createClassLoaderForTask(Project project, Project bootstrapProject, String muzzleTaskName) {
final List<URL> userUrls = new ArrayList<>()
project.getLogger().info("Creating task classpath")
project.configurations.getByName(muzzleTaskName).resolvedConfiguration.files.each { File jarFile ->
project.getLogger().info("-- Added to instrumentation classpath: $jarFile")
userUrls.add(jarFile.toURI().toURL())
}
for (File f : bootstrapProject.sourceSets.main.runtimeClasspath.getFiles()) {
project.getLogger().info("-- Added to instrumentation bootstrap classpath: $f")
userUrls.add(f.toURI().toURL())
}
return new URLClassLoader(userUrls.toArray(new URL[0]), (ClassLoader) null)
}
/**
* Convert a muzzle directive to a list of artifacts
*/
private static List<Artifact> muzzleDirectiveToArtifacts(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
final Artifact directiveArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", muzzleDirective.versions)
final VersionRangeRequest rangeRequest = new VersionRangeRequest()
rangeRequest.setRepositories(MUZZLE_REPOS)
rangeRequest.setArtifact(directiveArtifact)
final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
final List<Artifact> allVersionArtifacts = filterVersion(rangeResult.versions).collect { version ->
new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", version.toString())
}
return allVersionArtifacts
}
/**
* Create a list of muzzle directives which assert the opposite of the given MuzzleDirective.
*/
private static List<MuzzleDirective> inverseOf(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
List<MuzzleDirective> inverseDirectives = new ArrayList<>()
final Artifact allVerisonsArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", "[,)")
final Artifact directiveArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", muzzleDirective.versions)
final VersionRangeRequest allRangeRequest = new VersionRangeRequest()
allRangeRequest.setRepositories(MUZZLE_REPOS)
allRangeRequest.setArtifact(allVerisonsArtifact)
final VersionRangeResult allRangeResult = system.resolveVersionRange(session, allRangeRequest)
final VersionRangeRequest rangeRequest = new VersionRangeRequest()
rangeRequest.setRepositories(MUZZLE_REPOS)
rangeRequest.setArtifact(directiveArtifact)
final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
filterVersion(allRangeResult.versions).collect { version ->
if (!rangeResult.versions.contains(version)) {
final MuzzleDirective inverseDirective = new MuzzleDirective()
inverseDirective.group = muzzleDirective.group
inverseDirective.module = muzzleDirective.module
inverseDirective.versions = "$version"
inverseDirective.assertPass = !muzzleDirective.assertPass
inverseDirectives.add(inverseDirective)
}
}
return inverseDirectives
}
/**
* Configure a muzzle task to pass or fail a given version.
*
* @param assertPass If true, assert that muzzle validation passes
* @param versionArtifact version to assert against.
* @param instrumentationProject instrumentation being asserted against.
* @param runAfter Task which runs before the new muzzle task.
* @param bootstrapProject Agent bootstrap project.
* @param toolingProject Agent tooling project.
*
* @return The created muzzle task.
*/
private static Task addMuzzleTask(MuzzleDirective muzzleDirective, Artifact versionArtifact, Project instrumentationProject, Task runAfter, Project bootstrapProject, Project toolingProject) {
def taskName = "muzzle-Assert${muzzleDirective.assertPass ? "Pass" : "Fail"}-$versionArtifact.groupId-$versionArtifact.artifactId-$versionArtifact.version"
def config = instrumentationProject.configurations.create(taskName)
config.dependencies.add(instrumentationProject.dependencies.create("$versionArtifact.groupId:$versionArtifact.artifactId:$versionArtifact.version") {
transitive = true
})
for (String additionalDependency : muzzleDirective.additionalDependencies) {
config.dependencies.add(instrumentationProject.dependencies.create(additionalDependency) {
transitive = true
})
}
def muzzleTask = instrumentationProject.task(taskName) {
doLast {
final ClassLoader userCL = createClassLoaderForTask(instrumentationProject, bootstrapProject, taskName)
final ClassLoader agentCL = createDDClassloader(instrumentationProject, toolingProject)
// find all instrumenters, get muzzle, and assert
Method assertionMethod = agentCL.loadClass('datadog.trace.agent.tooling.muzzle.MuzzleVersionScanPlugin')
.getMethod('assertInstrumentationMuzzled', ClassLoader.class, boolean.class)
assertionMethod.invoke(null, userCL, muzzleDirective.assertPass)
}
}
runAfter.finalizedBy(muzzleTask)
return muzzleTask
}
/**
* Create muzzle's repository system
*/
private static RepositorySystem newRepositorySystem() {
DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator()
locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class)
locator.addService(TransporterFactory.class, HttpTransporterFactory.class)
return locator.getService(RepositorySystem.class)
}
/**
* Create muzzle's repository system session
*/
private static RepositorySystemSession newRepositorySystemSession(RepositorySystem system) {
DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession()
def tempDir = File.createTempDir()
tempDir.deleteOnExit()
LocalRepository localRepo = new LocalRepository(tempDir)
session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo))
return session
}
/**
* Filter out snapshot-type builds from versions list.
*/
private static filterVersion(List<Version> list) {
list.removeIf {
def version = it.toString().toLowerCase()
return version.contains("rc") ||
version.contains(".cr") ||
version.contains("alpha") ||
version.contains("beta") ||
version.contains("-b") ||
version.contains(".m") ||
version.contains("-dev") ||
version.contains("public_draft")
}
return list
}
}
// plugin extension classes
/**
* A pass or fail directive for a single dependency.
*/
class MuzzleDirective {
String group
String module
String versions
List<String> additionalDependencies = new ArrayList<>()
boolean assertPass
boolean assertInverse = false
void extraDependency(String compileString) {
additionalDependencies.add(compileString)
}
}
/**
* Muzzle extension containing all pass and fail directives.
*/
class MuzzleExtension {
final List<MuzzleDirective> directives = new ArrayList<>()
private final ObjectFactory objectFactory
@javax.inject.Inject
MuzzleExtension(final ObjectFactory objectFactory) {
this.objectFactory = objectFactory
}
void pass(Action<? super MuzzleDirective> action) {
final MuzzleDirective pass = objectFactory.newInstance(MuzzleDirective)
action.execute(pass)
pass.assertPass = true
directives.add(pass)
}
void fail(Action<? super MuzzleDirective> action) {
final MuzzleDirective fail = objectFactory.newInstance(MuzzleDirective)
action.execute(fail)
fail.assertPass = false
directives.add(fail)
}
}

View File

@ -2,9 +2,12 @@ package datadog.trace.agent.tooling.muzzle;
import datadog.trace.agent.tooling.HelperInjector;
import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.bootstrap.WeakMap;
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.List;
import java.util.ServiceLoader;
import java.util.WeakHashMap;
/**
* Entry point for muzzle version scan gradle plugin.
@ -15,7 +18,19 @@ import java.util.ServiceLoader;
* <p>Additionally, after a successful muzzle validation run each instrumenter's helper injector.
*/
public class MuzzleVersionScanPlugin {
public static void assertInstrumentationNotMuzzled(ClassLoader cl) throws Exception {
static {
// prevent WeakMap from logging warning while plugin is running
WeakMap.Provider.registerIfAbsent(
new WeakMap.Supplier() {
@Override
public <K, V> WeakMap<K, V> get() {
return new WeakMap.MapAdapter<>(Collections.synchronizedMap(new WeakHashMap<K, V>()));
}
});
}
public static void assertInstrumentationMuzzled(ClassLoader cl, boolean assertPass)
throws Exception {
// muzzle validate all instrumenters
for (Instrumenter instrumenter :
ServiceLoader.load(Instrumenter.class, MuzzleGradlePlugin.class.getClassLoader())) {
@ -36,7 +51,12 @@ public class MuzzleVersionScanPlugin {
m.setAccessible(true);
ReferenceMatcher muzzle = (ReferenceMatcher) m.invoke(instrumenter);
List<Reference.Mismatch> mismatches = muzzle.getMismatchedReferenceSources(cl);
if (mismatches.size() > 0) {
boolean passed = mismatches.size() == 0;
if (mismatches.size() > 0) {}
if (passed && !assertPass) {
System.err.println("MUZZLE PASSED BUT FAILURE WAS EXPECTED");
throw new RuntimeException("Instrumentation unexpectedly passed Muzzle validation");
} else if (!passed && assertPass) {
System.err.println(
"FAILED MUZZLE VALIDATION: " + instrumenter.getClass().getName() + " mismatches:");
for (Reference.Mismatch mismatch : mismatches) {
@ -51,6 +71,7 @@ public class MuzzleVersionScanPlugin {
}
}
// run helper injector on all instrumenters
if (assertPass) {
for (Instrumenter instrumenter :
ServiceLoader.load(Instrumenter.class, MuzzleGradlePlugin.class.getClassLoader())) {
if (instrumenter.getClass().getName().endsWith("TraceConfigInstrumentation")) {
@ -81,6 +102,7 @@ public class MuzzleVersionScanPlugin {
}
}
}
}
public static void printMuzzleReferences() {
for (Instrumenter instrumenter :

View File

@ -16,6 +16,24 @@ compileLagomTestGroovy {
targetCompatibility = 1.8
}
muzzle {
pass {
group = 'com.typesafe.akka'
module = 'akka-http_2.11'
versions = "[10.0.0,)"
// later versions of akka-http expect streams to be provided
extraDependency 'com.typesafe.akka:akka-stream_2.11:2.4.14'
}
pass {
group = 'com.typesafe.akka'
module = 'akka-http_2.12'
versions = "[10.0.0,)"
// later versions of akka-http expect streams to be provided
extraDependency 'com.typesafe.akka:akka-stream_2.12:2.4.14'
}
}
dependencies {
compileOnly group: 'com.typesafe.akka', name: 'akka-http_2.11', version: '10.0.0'

View File

@ -1,28 +1,18 @@
apply plugin: 'version-scan'
apply from: "${rootDir}/gradle/java.gradle"
versionScan {
muzzle {
fail {
group = "commons-httpclient"
module = "commons-httpclient"
versions = "[,4.3)"
}
pass {
group = "org.apache.httpcomponents"
module = "httpclient"
versions = "[4.3,)"
legacyGroup = "commons-httpclient"
legacyModule = "commons-httpclient"
verifyPresent = [
// The commented out classes are required for the instrumentation (this is checked by our bytebuddy rules).
// Once the verifier can report on the failed versions these classes can be commented out and the pass range can be widened.
// "org.apache.http.HttpException" : null,
// "org.apache.http.HttpRequest" : null,
// "org.apache.http.client.RedirectStrategy" : null,
"org.apache.http.client.methods.CloseableHttpResponse": null,
"org.apache.http.client.methods.HttpExecutionAware" : null,
"org.apache.http.client.methods.HttpRequestWrapper" : null,
"org.apache.http.client.protocol.HttpClientContext" : null,
// "org.apache.http.conn.routing.HttpRoute" : null,
"org.apache.http.impl.execchain.ClientExecChain" : null
]
assertInverse = true
}
}
apply from: "${rootDir}/gradle/java.gradle"
apply plugin: 'org.unbroken-dome.test-sets'

View File

@ -1,7 +1,6 @@
package datadog.trace.instrumentation.apachehttpclient;
import static datadog.trace.agent.tooling.ByteBuddyElementMatchers.safeHasSuperType;
import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses;
import static io.opentracing.log.Fields.ERROR_OBJECT;
import static net.bytebuddy.matcher.ElementMatchers.isAbstract;
import static net.bytebuddy.matcher.ElementMatchers.isMethod;
@ -36,22 +35,6 @@ public class ApacheHttpClientInstrumentation extends Instrumenter.Default {
.or(safeHasSuperType(named("org.apache.http.impl.client.CloseableHttpClient")));
}
@Override
public ElementMatcher<ClassLoader> classLoaderMatcher() {
return classLoaderHasClasses(
"org.apache.http.HttpException",
"org.apache.http.HttpRequest",
"org.apache.http.client.RedirectStrategy",
"org.apache.http.client.methods.CloseableHttpResponse",
"org.apache.http.client.methods.HttpExecutionAware",
"org.apache.http.client.methods.HttpRequestWrapper",
"org.apache.http.client.protocol.HttpClientContext",
"org.apache.http.conn.routing.HttpRoute",
"org.apache.http.impl.execchain.ClientExecChain",
"org.apache.http.impl.client.CloseableHttpClient",
"org.apache.http.impl.client.InternalHttpClient");
}
@Override
public String[] helperClassNames() {
return new String[] {

View File

@ -7,6 +7,19 @@ testSets {
latestDepTest
}
muzzle {
pass {
group = 'com.typesafe.play'
module = 'play_2.11'
versions = '[2.4.0,2.7.0-M1)'
}
pass {
group = 'com.typesafe.play'
module = 'play_2.12'
versions = '[2.4.0,2.7.0-M1)'
}
}
dependencies {
compileOnly group: 'com.typesafe.play', name: 'play_2.11', version: '2.4.0'

View File

@ -1,8 +1,6 @@
package datadog.trace.instrumentation.play;
import static datadog.trace.agent.tooling.ByteBuddyElementMatchers.safeHasSuperType;
import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClassWithMethod;
import static datadog.trace.agent.tooling.ClassLoaderMatcher.classLoaderHasClasses;
import static io.opentracing.log.Fields.ERROR_OBJECT;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.returns;
@ -50,18 +48,6 @@ public final class PlayInstrumentation extends Instrumenter.Default {
return safeHasSuperType(named("play.api.mvc.Action"));
}
@Override
public ElementMatcher<ClassLoader> classLoaderMatcher() {
return classLoaderHasClasses(
"akka.japi.JavaPartialFunction",
"play.api.mvc.Action",
"play.api.mvc.Result",
"scala.Option",
"scala.Tuple2",
"scala.concurrent.Future")
.and(classLoaderHasClassWithMethod("play.api.mvc.Request", "tags"));
}
@Override
public String[] helperClassNames() {
return new String[] {