diff --git a/controllers/storage.go b/controllers/storage.go
index 5059df8f..748faf07 100644
--- a/controllers/storage.go
+++ b/controllers/storage.go
@@ -100,6 +100,11 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
 	dir := filepath.Dir(artifact.Path)
 	var errors []string
 	_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			errors = append(errors, err.Error())
+			return nil
+		}
+
 		if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
 			if err := os.Remove(path); err != nil {
 				errors = append(errors, info.Name())
diff --git a/controllers/storage_test.go b/controllers/storage_test.go
index 372c9e0f..268c4488 100644
--- a/controllers/storage_test.go
+++ b/controllers/storage_test.go
@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"testing"
 	"time"
@@ -250,3 +251,22 @@ func TestArchiveIgnore(t *testing.T) {
 		testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
 	})
 }
+
+func TestStorageRemoveAllButCurrent(t *testing.T) {
+	t.Run("bad directory in archive", func(t *testing.T) {
+		dir, err := ioutil.TempDir("", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { os.RemoveAll(dir) })
+
+		s, err := NewStorage(dir, "hostname", time.Minute)
+		if err != nil {
+			t.Fatalf("Valid path did not successfully return: %v", err)
+		}
+
+		if err := s.RemoveAllButCurrent(sourcev1.Artifact{Path: path.Join(dir, "really", "nonexistent")}); err == nil {
+			t.Fatal("Did not error while pruning non-existent path")
+		}
+	})
+}