diff --git a/libmachine/host.go b/libmachine/host.go
index 5f28861ff7..75cd2d8876 100644
--- a/libmachine/host.go
+++ b/libmachine/host.go
@@ -2,6 +2,7 @@ package libmachine
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -21,8 +22,9 @@ import (
 )
 
 var (
-	validHostNameChars   = `[a-zA-Z0-9\-\.]`
-	validHostNamePattern = regexp.MustCompile(`^` + validHostNameChars + `+$`)
+	validHostNameChars                = `[a-zA-Z0-9\-\.]`
+	validHostNamePattern              = regexp.MustCompile(`^` + validHostNameChars + `+$`)
+	errMachineMustBeRunningForUpgrade = errors.New("Error: machine must be running to upgrade.")
 )
 
 type Host struct {
@@ -243,6 +245,15 @@ func (h *Host) Restart() error {
 }
 
 func (h *Host) Upgrade() error {
+	machineState, err := h.Driver.GetState()
+	if err != nil {
+		return err
+	}
+
+	if machineState != state.Running {
+		log.Fatal(errMachineMustBeRunningForUpgrade)
+	}
+
 	provisioner, err := provision.DetectProvisioner(h.Driver)
 	if err != nil {
 		return err
diff --git a/test/integration/driver-virtualbox.bats b/test/integration/driver-virtualbox.bats
index ed04adff0a..eec4cf6483 100644
--- a/test/integration/driver-virtualbox.bats
+++ b/test/integration/driver-virtualbox.bats
@@ -119,6 +119,11 @@ buildMachineWithOldIsoCheckUpgrade() {
   [[ ${lines[1]} == *"Stopped"*  ]]
 }
 
+@test "$DRIVER: machine should not allow upgrade when stopped" {
+  run machine upgrade $NAME
+  [[ "$status" -eq 1 ]]
+}
+
 @test "$DRIVER: start" {
   run machine start $NAME
   [ "$status" -eq 0  ]