diff --git a/go/src/bashbrew/config.go b/go/src/bashbrew/config.go
index c78d218..061987b 100644
--- a/go/src/bashbrew/config.go
+++ b/go/src/bashbrew/config.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/codegangsta/cli"
 	"github.com/docker-library/go-dockerlibrary/pkg/stripper"
@@ -98,7 +99,7 @@ func (config FlagsConfig) ApplyTo(cmd string, c *cli.Context) error {
 		vars["global"] = nil
 	}
 	for key, val := range vars["global"] {
-		if !c.GlobalIsSet(key) {
+		if !c.GlobalIsSet(key) && !isEnvVarSet(flagEnvVars[key]) {
 			switch val.(type) {
 			case string:
 				strVal := val.(string)
@@ -120,7 +121,7 @@ func (config FlagsConfig) ApplyTo(cmd string, c *cli.Context) error {
 		}
 	}
 	for key, val := range vars["local"] {
-		if !c.IsSet(key) {
+		if !c.IsSet(key) && !isEnvVarSet(flagEnvVars[key]) {
 			switch val.(type) {
 			case string:
 				strVal := val.(string)
@@ -144,6 +145,23 @@ func (config FlagsConfig) ApplyTo(cmd string, c *cli.Context) error {
 	return nil
 }
 
+// https://github.com/urfave/cli/blob/73aa67b7a20db7514b1c086866469c49bc93a569/altsrc/flag.go#L237-L251
+func isEnvVarSet(envVars string) bool {
+	for _, envVar := range strings.Split(envVars, ",") {
+		envVar = strings.TrimSpace(envVar)
+		if envVal := os.Getenv(envVar); envVal != "" {
+			// TODO: Can't use this for bools as
+			// set means that it was true or false based on
+			// Bool flag type, should work for other types
+			if len(envVal) > 0 {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 func (ce FlagsConfigEntry) String() string {
 	buf := &bytes.Buffer{}
 	e, err := control.NewEncoder(buf)
diff --git a/go/src/bashbrew/main.go b/go/src/bashbrew/main.go
index f586e00..1008dcd 100644
--- a/go/src/bashbrew/main.go
+++ b/go/src/bashbrew/main.go
@@ -26,6 +26,14 @@ var (
 
 	debugFlag  = false
 	noSortFlag = false
+
+	// separated so that FlagsConfig.ApplyTo can access them
+	flagEnvVars = map[string]string{
+		"debug":   "BASHBREW_DEBUG",
+		"config":  "BASHBREW_CONFIG",
+		"library": "BASHBREW_LIBRARY",
+		"cache":   "BASHBREW_CACHE",
+	}
 )
 
 func initDefaultConfigPath() string {
@@ -144,7 +152,7 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.BoolFlag{
 			Name:   "debug",
-			EnvVar: "BASHBREW_DEBUG",
+			EnvVar: flagEnvVars["debug"],
 			Usage:  `enable more output (esp. all "docker build" output instead of only output on failure)`,
 		},
 		cli.BoolFlag{
@@ -164,19 +172,19 @@ func main() {
 		cli.StringFlag{
 			Name:   "config",
 			Value:  initDefaultConfigPath(),
-			EnvVar: "BASHBREW_CONFIG",
+			EnvVar: flagEnvVars["config"],
 			Usage:  `where default "flags" configuration can be overridden more persistently`,
 		},
 		cli.StringFlag{
 			Name:   "library",
 			Value:  filepath.Join(os.Getenv("HOME"), "docker", "official-images", "library"),
-			EnvVar: "BASHBREW_LIBRARY",
+			EnvVar: flagEnvVars["library"],
 			Usage:  "where the bodies are buried",
 		},
 		cli.StringFlag{
 			Name:   "cache",
 			Value:  initDefaultCachePath(),
-			EnvVar: "BASHBREW_CACHE",
+			EnvVar: flagEnvVars["cache"],
 			Usage:  "where the git wizardry is stashed",
 		},
 	}