diff --git a/dd-smoke-tests/README.md b/dd-smoke-tests/README.md
index 242579d894..7e8212d37a 100644
--- a/dd-smoke-tests/README.md
+++ b/dd-smoke-tests/README.md
@@ -1,8 +1,8 @@
 # Datadog Smoke Tests
-Assert that various application servers will start up with the Datadog JavaAgent without any obvious ill effects.
+Assert that various applications will start up with the Datadog JavaAgent without any obvious ill effects.
 
 Each subproject underneath `dd-smoke-tests` is a single smoke test. Each test does the following
-* Launch the app server with stdout and stderr logged to `$buildDir/reports/server.log`
-* Run a spock test which does 200 requests to an endpoint on the server and asserts on an expected response.
+* Launch the application with stdout and stderr logged to `$buildDir/reports/server.log`
+* For web servers, run a spock test which does 200 requests to an endpoint on the server and asserts on an expected response.
 
 Note that there is nothing special about doing 200 requests. 200 is simply an arbitrarily large number to exercise the server.
diff --git a/dd-smoke-tests/cli/cli.gradle b/dd-smoke-tests/cli/cli.gradle
new file mode 100644
index 0000000000..84b0676780
--- /dev/null
+++ b/dd-smoke-tests/cli/cli.gradle
@@ -0,0 +1,23 @@
+plugins {
+  id "com.github.johnrengelman.shadow" version "4.0.4"
+}
+apply from: "${rootDir}/gradle/java.gradle"
+description = 'Command Line Application Smoke Tests.'
+
+jar {
+  manifest {
+    attributes(
+      'Main-Class': 'datadog.smoketest.cli.CliApplication'
+    )
+  }
+}
+
+dependencies {
+  testCompile project(':dd-smoke-tests')
+}
+
+tasks.withType(Test).configureEach {
+  dependsOn shadowJar
+
+  jvmArgs "-Ddatadog.smoketest.cli.shadowJar.path=${tasks.shadowJar.archivePath}"
+}
diff --git a/dd-smoke-tests/cli/src/main/java/datadog/smoketest/cli/CliApplication.java b/dd-smoke-tests/cli/src/main/java/datadog/smoketest/cli/CliApplication.java
new file mode 100644
index 0000000000..65e8428d9e
--- /dev/null
+++ b/dd-smoke-tests/cli/src/main/java/datadog/smoketest/cli/CliApplication.java
@@ -0,0 +1,16 @@
+package datadog.smoketest.cli;
+
+/** Simple application that sleeps for a bit than quits. */
+public class CliApplication {
+
+  /** @param args Only argument is the sleep delay (in seconds) */
+  public static void main(final String[] args) throws InterruptedException {
+    final int delay = Integer.parseInt(args[0]);
+
+    System.out.println("Going to shut down after " + delay + "seconds");
+
+    Thread.sleep(delay * 1000);
+
+    System.out.println("Shutting down");
+  }
+}
diff --git a/dd-smoke-tests/cli/src/test/groovy/datadog/smoketest/CliApplicationSmokeTest.groovy b/dd-smoke-tests/cli/src/test/groovy/datadog/smoketest/CliApplicationSmokeTest.groovy
new file mode 100644
index 0000000000..3c6e0b9aac
--- /dev/null
+++ b/dd-smoke-tests/cli/src/test/groovy/datadog/smoketest/CliApplicationSmokeTest.groovy
@@ -0,0 +1,34 @@
+package datadog.smoketest
+
+import spock.util.concurrent.PollingConditions
+
+class CliApplicationSmokeTest extends AbstractSmokeTest {
+  // Estimate for the amount of time instrumentation takes plus a little extra
+  private static final int INSTRUMENTATION_DELAY = 6 + 5
+
+  private static final int SHUTDOWN_DELAY = 2
+  
+  @Override
+  ProcessBuilder createProcessBuilder() {
+    String cliShadowJar = System.getProperty("datadog.smoketest.cli.shadowJar.path")
+
+    List<String> command = new ArrayList<>()
+    command.add(javaPath())
+    command.addAll(defaultJavaProperties)
+    command.addAll((String[]) ["-jar", cliShadowJar, String.valueOf(SHUTDOWN_DELAY)])
+    ProcessBuilder processBuilder = new ProcessBuilder(command)
+    processBuilder.directory(new File(buildDirectory))
+  }
+
+  def "Cli application process ends before timeout"() {
+    setup:
+    def conditions = new PollingConditions(timeout: INSTRUMENTATION_DELAY, initialDelay: SHUTDOWN_DELAY)
+
+    expect:
+    serverProcess.isAlive()
+
+    conditions.eventually {
+      assert !serverProcess.isAlive()
+    }
+  }
+}
diff --git a/settings.gradle b/settings.gradle
index dcd99258c7..6441d5e934 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -22,6 +22,7 @@ include ':utils:gc-utils'
 include ':dd-smoke-tests:play'
 include ':dd-smoke-tests:springboot'
 include ':dd-smoke-tests:wildfly'
+include ':dd-smoke-tests:cli'
 
 // instrumentation:
 include ':dd-java-agent:instrumentation:akka-http-10.0'