diff --git a/config/settings.yml b/config/settings.yml
index f9f6b9a9..ab3675df 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -412,6 +412,7 @@ discourse_ai:
 
   ai_spam_detection_enabled:
     default: false
+    validator: "DiscourseAi::Configuration::SpamDetectionValidator"
   ai_spam_detection_user_id:
     default: ""
     hidden: true
diff --git a/lib/configuration/spam_detection_validator.rb b/lib/configuration/spam_detection_validator.rb
index 4201dcc4..6201cf3d 100644
--- a/lib/configuration/spam_detection_validator.rb
+++ b/lib/configuration/spam_detection_validator.rb
@@ -8,7 +8,8 @@ module DiscourseAi
       end
 
       def valid_value?(val)
-        return true if Rails.env.test?
+        # only validate when enabling spam detection
+        return true if val == "f" || val == "false"
         return true if AiModerationSetting.spam
 
         false
diff --git a/spec/configuration/spam_detection_validator_spec.rb b/spec/configuration/spam_detection_validator_spec.rb
new file mode 100644
index 00000000..bd359254
--- /dev/null
+++ b/spec/configuration/spam_detection_validator_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Configuration::SpamDetectionValidator do
+  let(:validator) { described_class.new }
+
+  it "always returns true if setting the value to false" do
+    expect(validator.valid_value?("f")).to eq(true)
+  end
+
+  context "when a moderation setting exists" do
+    fab!(:llm_model)
+    before { AiModerationSetting.create!(setting_type: "spam", llm_model_id: llm_model.id) }
+
+    it "returns true if a moderation setting for spam exists" do
+      expect(validator.valid_value?("t")).to eq(true)
+    end
+  end
+
+  context "when no moderation setting exists" do
+    it "returns false if a moderation setting for spam does not exist" do
+      expect(validator.valid_value?("t")).to eq(false)
+    end
+
+    it "returns an error message when no moderation setting exists" do
+      expect(validator.error_message).to eq(
+        I18n.t("discourse_ai.spam_detection.configuration_missing"),
+      )
+    end
+  end
+end