tests

app/jobs/regular/generate_inferred_concepts.rb

@@ -59,9 +59,9 @@ module Jobs
         end
       else
         if item_type == "topics"
-          DiscourseAi::InferredConcepts::Manager.analyze_topic(item)
+          DiscourseAi::InferredConcepts::Manager.generate_concepts_from_topic(item)
         else # posts
-          DiscourseAi::InferredConcepts::Manager.analyze_post(item)
+          DiscourseAi::InferredConcepts::Manager.generate_concepts_from_post(item)
         end
       end
     end

app/models/inferred_concept.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
 class InferredConcept < ActiveRecord::Base
-  has_and_belongs_to_many :topics
-  has_and_belongs_to_many :posts
+  has_many :inferred_concept_topics
+  has_many :topics, through: :inferred_concept_topics
+
+  has_many :inferred_concept_posts
+  has_many :posts, through: :inferred_concept_posts
 
   validates :name, presence: true, uniqueness: true
 end
@@ -19,4 +22,4 @@ end
 # Indexes
 #
 #  index_inferred_concepts_on_name  (name) UNIQUE
-#
+#

app/models/inferred_concept_post.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class InferredConceptPost < ActiveRecord::Base
+  self.table_name = "inferred_concepts_posts"
+
+  belongs_to :inferred_concept
+  belongs_to :post
+
+  validates :inferred_concept_id, presence: true
+  validates :post_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :post_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts_posts
+#
+#  inferred_concept_id :bigint           not null
+#  post_id             :bigint           not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_posts_uniqueness  (post_id,inferred_concept_id) UNIQUE
+#  index_inferred_concepts_posts_on_inferred_concept_id  (inferred_concept_id)
+#

app/models/inferred_concept_topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class InferredConceptTopic < ActiveRecord::Base
+  self.table_name = "inferred_concepts_topics"
+
+  belongs_to :inferred_concept
+  belongs_to :topic
+
+  validates :inferred_concept_id, presence: true
+  validates :topic_id, presence: true
+  validates :inferred_concept_id, uniqueness: { scope: :topic_id }
+end
+
+# == Schema Information
+#
+# Table name: inferred_concepts_topics
+#
+#  inferred_concept_id :bigint           not null
+#  topic_id            :bigint           not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#
+# Indexes
+#
+#  index_inferred_concepts_topics_uniqueness  (topic_id,inferred_concept_id) UNIQUE
+#  index_inferred_concepts_topics_on_inferred_concept_id  (inferred_concept_id)
+#

app/serializers/ai_inferred_concept_post_serializer.rb

@@ -12,23 +12,23 @@ class AiInferredConceptPostSerializer < ApplicationSerializer
              :excerpt,
              :truncated,
              :inferred_concepts
-             
+
   def avatar_template
     User.avatar_template(object.username, object.uploaded_avatar_id)
   end
-  
+
   def excerpt
     Post.excerpt(object.cooked)
   end
-  
+
   def truncated
     object.cooked.length > SiteSetting.post_excerpt_maxlength
   end
-  
+
   def inferred_concepts
     ActiveModel::ArraySerializer.new(
       object.inferred_concepts,
-      each_serializer: InferredConceptSerializer
+      each_serializer: InferredConceptSerializer,
     )
   end
-end
+end

app/serializers/inferred_concept_serializer.rb

@@ -2,4 +2,4 @@
 
 class InferredConceptSerializer < ApplicationSerializer
   attributes :id, :name, :created_at, :updated_at
-end
+end

db/fixtures/personas/603_ai_personas.rb

@@ -72,9 +72,13 @@ DiscourseAi::Personas::Persona.system_personas.each do |persona_class, id|
 
   persona.tools = tools.map { |name, value| [name, value] }
 
-  persona.response_format = instance.response_format
+  # Only set response_format if it's not defined as a method in the persona class
+  if !instance.class.instance_methods.include?(:response_format)
+    persona.response_format = instance.response_format
+  end
 
-  persona.examples = instance.examples
+  # Only set examples if it's not defined as a method in the persona class
+  persona.examples = instance.examples if !instance.class.instance_methods.include?(:examples)
 
   persona.system_prompt = instance.system_prompt
   persona.top_p = instance.top_p

db/migrate/20250508182047_create_inferred_concepts_table.rb

@@ -5,7 +5,7 @@ class CreateInferredConceptsTable < ActiveRecord::Migration[7.2]
       t.string :name, null: false
       t.timestamps
     end
-    
+
     add_index :inferred_concepts, :name, unique: true
   end
 end

lib/inferred_concepts/finder.rb

@@ -15,7 +15,7 @@ module DiscourseAi
             .find { |p| p.id == SiteSetting.inferred_concepts_generate_persona.to_i }
             .new
 
-        llm = LlmModel.find(persona.class.default_llm_id)
+        llm = LlmModel.find(persona.default_llm_id)
         context =
           DiscourseAi::Personas::BotContext.new(
             messages: [{ type: :user, content: content }],
@@ -79,7 +79,7 @@ module DiscourseAi
         # Exclude topics that already have concepts
         topics_with_concepts = <<~SQL
           SELECT DISTINCT topic_id
-          FROM topics_inferred_concepts
+          FROM inferred_concepts_topics
         SQL
 
         query = query.where("topics.id NOT IN (#{topics_with_concepts})")
@@ -129,7 +129,7 @@ module DiscourseAi
         # Exclude posts that already have concepts
         posts_with_concepts = <<~SQL
           SELECT DISTINCT post_id
-          FROM posts_inferred_concepts
+          FROM inferred_concepts_posts
         SQL
 
         query = query.where("posts.id NOT IN (#{posts_with_concepts})")
@@ -154,7 +154,7 @@ module DiscourseAi
             .find { |p| p.id == SiteSetting.inferred_concepts_deduplicate_persona.to_i }
             .new
 
-        llm = LlmModel.find(persona.class.default_llm_id)
+        llm = LlmModel.find(persona.default_llm_id)
 
         # Create the input for the deduplicator
         input = { type: :user, content: concept_names.join(", ") }

lib/post_extensions.rb

@@ -12,7 +12,8 @@ module DiscourseAi
                class_name: "ClassificationResult",
                as: :target
 
-      has_and_belongs_to_many :inferred_concepts
+      has_many :inferred_concept_posts
+      has_many :inferred_concepts, through: :inferred_concept_posts
     end
   end
 end

lib/topic_extensions.rb

@@ -11,8 +11,9 @@ module DiscourseAi
               -> { where(summary_type: AiSummary.summary_types[:gist]) },
               class_name: "AiSummary",
               as: :target
-              
-      has_and_belongs_to_many :inferred_concepts
+
+      has_many :inferred_concept_topics
+      has_many :inferred_concepts, through: :inferred_concept_topics
     end
   end
 end

spec/fabricators/inferred_concept_fabricator.rb

@@ -0,0 +1,2 @@
+# frozen_string_literal: true
+Fabricator(:inferred_concept) { name { sequence(:name) { |i| "concept_#{i}" } } }

spec/jobs/regular/generate_inferred_concepts_spec.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::GenerateInferredConcepts do
+  fab!(:topic) { Fabricate(:topic) }
+  fab!(:post) { Fabricate(:post) }
+  fab!(:concept) { Fabricate(:inferred_concept, name: "programming") }
+
+  before { SiteSetting.inferred_concepts_enabled = true }
+
+  describe "#execute" do
+    it "does nothing with blank item_ids" do
+      expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:match_topic_to_concepts)
+
+      subject.execute(item_type: "topics", item_ids: [])
+      subject.execute(item_type: "topics", item_ids: nil)
+    end
+
+    it "does nothing with blank item_type" do
+      expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:match_topic_to_concepts)
+
+      subject.execute(item_type: "", item_ids: [topic.id])
+      subject.execute(item_type: nil, item_ids: [topic.id])
+    end
+
+    it "validates item_type to be topics or posts" do
+      expect(Rails.logger).to receive(:error).with(/Invalid item_type/)
+
+      subject.execute(item_type: "invalid", item_ids: [1])
+    end
+
+    context "with topics" do
+      it "processes topics in match_only mode" do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_topic_to_concepts).with(
+          topic,
+        )
+
+        subject.execute(item_type: "topics", item_ids: [topic.id], match_only: true)
+      end
+
+      it "processes topics in generation mode" do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(
+          :generate_concepts_from_topic,
+        ).with(topic)
+
+        subject.execute(item_type: "topics", item_ids: [topic.id], match_only: false)
+      end
+
+      it "handles topics that don't exist" do
+        # Non-existent IDs should be silently skipped (no error expected)
+        expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:match_topic_to_concepts)
+
+        subject.execute(
+          item_type: "topics",
+          item_ids: [999_999], # non-existent ID
+          match_only: true,
+        )
+      end
+
+      it "processes multiple topics" do
+        topic2 = Fabricate(:topic)
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_topic_to_concepts).with(
+          topic,
+        )
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_topic_to_concepts).with(
+          topic2,
+        )
+
+        subject.execute(item_type: "topics", item_ids: [topic.id, topic2.id], match_only: true)
+      end
+
+      it "processes topics in batches" do
+        topics = Array.new(5) { Fabricate(:topic) }
+        topic_ids = topics.map(&:id)
+
+        # Should process in batches of 3
+        expect(Topic).to receive(:where).with(id: topic_ids[0..2]).and_call_original
+        expect(Topic).to receive(:where).with(id: topic_ids[3..4]).and_call_original
+
+        subject.execute(item_type: "topics", item_ids: topic_ids, batch_size: 3, match_only: true)
+      end
+    end
+
+    context "with posts" do
+      it "processes posts in match_only mode" do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_post_to_concepts).with(
+          post,
+        )
+
+        subject.execute(item_type: "posts", item_ids: [post.id], match_only: true)
+      end
+
+      it "processes posts in generation mode" do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(
+          :generate_concepts_from_post,
+        ).with(post)
+
+        subject.execute(item_type: "posts", item_ids: [post.id], match_only: false)
+      end
+
+      it "handles posts that don't exist" do
+        # Non-existent IDs should be silently skipped (no error expected)
+        expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:match_post_to_concepts)
+
+        subject.execute(
+          item_type: "posts",
+          item_ids: [999_999], # non-existent ID
+          match_only: true,
+        )
+      end
+
+      it "processes multiple posts" do
+        post2 = Fabricate(:post)
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_post_to_concepts).with(
+          post,
+        )
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:match_post_to_concepts).with(
+          post2,
+        )
+
+        subject.execute(item_type: "posts", item_ids: [post.id, post2.id], match_only: true)
+      end
+    end
+
+    it "handles exceptions during processing" do
+      allow(DiscourseAi::InferredConcepts::Manager).to receive(:match_topic_to_concepts).and_raise(
+        StandardError.new("Test error"),
+      )
+
+      expect(Rails.logger).to receive(:error).with(
+        /Error generating concepts from topic #{topic.id}/,
+      )
+
+      subject.execute(item_type: "topics", item_ids: [topic.id], match_only: true)
+    end
+
+    it "uses default batch size of 100" do
+      topics = Array.new(150) { Fabricate(:topic) }
+      topic_ids = topics.map(&:id)
+
+      # Should process in batches of 100
+      expect(Topic).to receive(:where).with(id: topic_ids[0..99]).and_call_original
+      expect(Topic).to receive(:where).with(id: topic_ids[100..149]).and_call_original
+
+      subject.execute(item_type: "topics", item_ids: topic_ids, match_only: true)
+    end
+
+    it "respects custom batch size" do
+      topics = Array.new(5) { Fabricate(:topic) }
+      topic_ids = topics.map(&:id)
+
+      # Should process in batches of 2
+      expect(Topic).to receive(:where).with(id: topic_ids[0..1]).and_call_original
+      expect(Topic).to receive(:where).with(id: topic_ids[2..3]).and_call_original
+      expect(Topic).to receive(:where).with(id: topic_ids[4..4]).and_call_original
+
+      subject.execute(item_type: "topics", item_ids: topic_ids, batch_size: 2, match_only: true)
+    end
+  end
+end

spec/jobs/scheduled/generate_concepts_from_popular_items_spec.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::GenerateConceptsFromPopularItems do
+  fab!(:topic) { Fabricate(:topic, posts_count: 6, views: 150, like_count: 12) }
+  fab!(:post) { Fabricate(:post, like_count: 8, post_number: 2) }
+
+  before do
+    SiteSetting.inferred_concepts_enabled = true
+    SiteSetting.inferred_concepts_daily_topics_limit = 20
+    SiteSetting.inferred_concepts_daily_posts_limit = 30
+    SiteSetting.inferred_concepts_min_posts = 5
+    SiteSetting.inferred_concepts_min_likes = 10
+    SiteSetting.inferred_concepts_min_views = 100
+    SiteSetting.inferred_concepts_post_min_likes = 5
+    SiteSetting.inferred_concepts_lookback_days = 30
+    SiteSetting.inferred_concepts_background_match = false
+  end
+
+  describe "#execute" do
+    it "does nothing when inferred_concepts_enabled is false" do
+      SiteSetting.inferred_concepts_enabled = false
+
+      expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:find_candidate_topics)
+      expect(DiscourseAi::InferredConcepts::Manager).not_to receive(:find_candidate_posts)
+      expect(Jobs).not_to receive(:enqueue)
+
+      subject.execute({})
+    end
+
+    it "processes popular topics when enabled" do
+      candidate_topics = [topic]
+
+      freeze_time do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).with(
+          limit: 20,
+          min_posts: 5,
+          min_likes: 10,
+          min_views: 100,
+          created_after: 30.days.ago,
+        ).and_return(candidate_topics)
+
+        expect(Jobs).to receive(:enqueue).with(
+          :generate_inferred_concepts,
+          item_type: "topics",
+          item_ids: [topic.id],
+          batch_size: 10,
+        )
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).and_return(
+          [],
+        )
+
+        subject.execute({})
+      end
+    end
+
+    it "processes popular posts when enabled" do
+      candidate_posts = [post]
+
+      freeze_time do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(
+          :find_candidate_topics,
+        ).and_return([])
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).with(
+          limit: 30,
+          min_likes: 5,
+          exclude_first_posts: true,
+          created_after: 30.days.ago,
+        ).and_return(candidate_posts)
+
+        expect(Jobs).to receive(:enqueue).with(
+          :generate_inferred_concepts,
+          item_type: "posts",
+          item_ids: [post.id],
+          batch_size: 10,
+        )
+
+        subject.execute({})
+      end
+    end
+
+    it "schedules background matching jobs when enabled" do
+      SiteSetting.inferred_concepts_background_match = true
+
+      candidate_topics = [topic]
+      candidate_posts = [post]
+
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).and_return(
+        candidate_topics,
+      )
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).and_return(
+        candidate_posts,
+      )
+
+      # Expect generation jobs
+      expect(Jobs).to receive(:enqueue).with(
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: [topic.id],
+        batch_size: 10,
+      )
+
+      expect(Jobs).to receive(:enqueue).with(
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: [post.id],
+        batch_size: 10,
+      )
+
+      # Expect background matching jobs
+      expect(Jobs).to receive(:enqueue_in).with(
+        1.hour,
+        :generate_inferred_concepts,
+        item_type: "topics",
+        item_ids: [topic.id],
+        batch_size: 10,
+        match_only: true,
+      )
+
+      expect(Jobs).to receive(:enqueue_in).with(
+        1.hour,
+        :generate_inferred_concepts,
+        item_type: "posts",
+        item_ids: [post.id],
+        batch_size: 10,
+        match_only: true,
+      )
+
+      subject.execute({})
+    end
+
+    it "does not schedule jobs when no candidates found" do
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).and_return(
+        [],
+      )
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).and_return(
+        [],
+      )
+
+      expect(Jobs).not_to receive(:enqueue)
+      expect(Jobs).not_to receive(:enqueue_in)
+
+      subject.execute({})
+    end
+
+    it "uses site setting values for topic filtering" do
+      SiteSetting.inferred_concepts_daily_topics_limit = 50
+      SiteSetting.inferred_concepts_min_posts = 8
+      SiteSetting.inferred_concepts_min_likes = 15
+      SiteSetting.inferred_concepts_min_views = 200
+      SiteSetting.inferred_concepts_lookback_days = 45
+
+      freeze_time do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).with(
+          limit: 50,
+          min_posts: 8,
+          min_likes: 15,
+          min_views: 200,
+          created_after: 45.days.ago,
+        ).and_return([])
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).and_return(
+          [],
+        )
+
+        subject.execute({})
+      end
+    end
+
+    it "uses site setting values for post filtering" do
+      SiteSetting.inferred_concepts_daily_posts_limit = 40
+      SiteSetting.inferred_concepts_post_min_likes = 8
+      SiteSetting.inferred_concepts_lookback_days = 45
+
+      freeze_time do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(
+          :find_candidate_topics,
+        ).and_return([])
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).with(
+          limit: 40,
+          min_likes: 8,
+          exclude_first_posts: true,
+          created_after: 45.days.ago,
+        ).and_return([])
+
+        subject.execute({})
+      end
+    end
+
+    it "handles nil site setting values gracefully" do
+      SiteSetting.inferred_concepts_daily_topics_limit = nil
+      SiteSetting.inferred_concepts_daily_posts_limit = nil
+      SiteSetting.inferred_concepts_min_posts = nil
+      SiteSetting.inferred_concepts_min_likes = nil
+      SiteSetting.inferred_concepts_min_views = nil
+      SiteSetting.inferred_concepts_post_min_likes = nil
+      # Keep lookback_days at default so .days.ago doesn't fail
+
+      freeze_time do
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).with(
+          limit: 0, # nil becomes 0
+          min_posts: 0, # nil becomes 0
+          min_likes: 0, # nil becomes 0
+          min_views: 0, # nil becomes 0
+          created_after: 30.days.ago, # default from before block
+        ).and_return([])
+
+        expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).with(
+          limit: 0, # nil becomes 0
+          min_likes: 0, # nil becomes 0
+          exclude_first_posts: true,
+          created_after: 30.days.ago, # default from before block
+        ).and_return([])
+
+        subject.execute({})
+      end
+    end
+
+    it "processes both topics and posts in the same run" do
+      candidate_topics = [topic]
+      candidate_posts = [post]
+
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_topics).and_return(
+        candidate_topics,
+      )
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:find_candidate_posts).and_return(
+        candidate_posts,
+      )
+
+      expect(Jobs).to receive(:enqueue).twice
+
+      subject.execute({})
+    end
+  end
+
+  context "job scheduling" do
+    it "is scheduled to run daily" do
+      expect(described_class.every).to eq(1.day)
+    end
+  end
+end

spec/lib/inferred_concepts/applier_spec.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::InferredConcepts::Applier do
+  fab!(:topic) { Fabricate(:topic, title: "Ruby Programming Tutorial") }
+  fab!(:post) { Fabricate(:post, raw: "This post is about advanced testing techniques") }
+  fab!(:user) { Fabricate(:user, username: "dev_user") }
+  fab!(:concept1) { Fabricate(:inferred_concept, name: "programming") }
+  fab!(:concept2) { Fabricate(:inferred_concept, name: "testing") }
+  fab!(:llm_model) { Fabricate(:fake_model) }
+
+  before do
+    SiteSetting.inferred_concepts_match_persona = -1
+    SiteSetting.inferred_concepts_enabled = true
+
+    # Set up the post's user
+    post.update!(user: user)
+  end
+
+  describe ".apply_to_topic" do
+    it "does nothing for blank topic or concepts" do
+      expect { described_class.apply_to_topic(nil, [concept1]) }.not_to raise_error
+      expect { described_class.apply_to_topic(topic, []) }.not_to raise_error
+      expect { described_class.apply_to_topic(topic, nil) }.not_to raise_error
+    end
+
+    it "associates concepts with topic" do
+      described_class.apply_to_topic(topic, [concept1, concept2])
+
+      expect(topic.inferred_concepts).to include(concept1, concept2)
+      expect(concept1.topics).to include(topic)
+      expect(concept2.topics).to include(topic)
+    end
+  end
+
+  describe ".apply_to_post" do
+    it "does nothing for blank post or concepts" do
+      expect { described_class.apply_to_post(nil, [concept1]) }.not_to raise_error
+      expect { described_class.apply_to_post(post, []) }.not_to raise_error
+      expect { described_class.apply_to_post(post, nil) }.not_to raise_error
+    end
+
+    it "associates concepts with post" do
+      described_class.apply_to_post(post, [concept1, concept2])
+
+      expect(post.inferred_concepts).to include(concept1, concept2)
+      expect(concept1.posts).to include(post)
+      expect(concept2.posts).to include(post)
+    end
+  end
+
+  describe ".topic_content_for_analysis" do
+    it "returns empty string for blank topic" do
+      expect(described_class.topic_content_for_analysis(nil)).to eq("")
+    end
+
+    it "extracts title and posts content" do
+      # Create additional posts for the topic
+      post1 = Fabricate(:post, topic: topic, post_number: 1, raw: "First post content", user: user)
+      post2 = Fabricate(:post, topic: topic, post_number: 2, raw: "Second post content", user: user)
+
+      content = described_class.topic_content_for_analysis(topic)
+
+      expect(content).to include(topic.title)
+      expect(content).to include("First post content")
+      expect(content).to include("Second post content")
+      expect(content).to include(user.username)
+      expect(content).to include("1)")
+      expect(content).to include("2)")
+    end
+
+    it "limits to first 10 posts" do
+      # Create 12 posts for the topic
+      12.times { |i| Fabricate(:post, topic: topic, post_number: i + 1, user: user) }
+
+      expect(Post).to receive(:where).with(topic_id: topic.id).and_call_original
+      expect_any_instance_of(ActiveRecord::Relation).to receive(:limit).with(10).and_call_original
+
+      described_class.topic_content_for_analysis(topic)
+    end
+  end
+
+  describe ".post_content_for_analysis" do
+    it "returns empty string for blank post" do
+      expect(described_class.post_content_for_analysis(nil)).to eq("")
+    end
+
+    it "extracts post content with topic context" do
+      content = described_class.post_content_for_analysis(post)
+
+      expect(content).to include(post.topic.title)
+      expect(content).to include(post.raw)
+      expect(content).to include(post.user.username)
+      expect(content).to include("Topic:")
+      expect(content).to include("Post by")
+    end
+
+    it "handles post without topic" do
+      # Mock the post to return nil for topic
+      allow(post).to receive(:topic).and_return(nil)
+
+      content = described_class.post_content_for_analysis(post)
+
+      expect(content).to include(post.raw)
+      expect(content).to include(post.user.username)
+      expect(content).to include("Topic: ")
+    end
+  end
+
+  describe ".match_existing_concepts" do
+    before do
+      allow(DiscourseAi::InferredConcepts::Manager).to receive(:list_concepts).and_return(
+        %w[programming testing ruby],
+      )
+    end
+
+    it "returns empty array for blank topic" do
+      expect(described_class.match_existing_concepts(nil)).to eq([])
+    end
+
+    it "returns empty array when no existing concepts" do
+      allow(DiscourseAi::InferredConcepts::Manager).to receive(:list_concepts).and_return([])
+
+      result = described_class.match_existing_concepts(topic)
+      expect(result).to eq([])
+    end
+
+    it "matches concepts and applies them to topic" do
+      expect(described_class).to receive(:topic_content_for_analysis).with(topic).and_return(
+        "content about programming",
+      )
+
+      expect(described_class).to receive(:match_concepts_to_content).with(
+        "content about programming",
+        %w[programming testing ruby],
+      ).and_return(["programming"])
+
+      expect(InferredConcept).to receive(:where).with(name: ["programming"]).and_return([concept1])
+
+      expect(described_class).to receive(:apply_to_topic).with(topic, [concept1])
+
+      result = described_class.match_existing_concepts(topic)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".match_existing_concepts_for_post" do
+    before do
+      allow(DiscourseAi::InferredConcepts::Manager).to receive(:list_concepts).and_return(
+        %w[programming testing ruby],
+      )
+    end
+
+    it "returns empty array for blank post" do
+      expect(described_class.match_existing_concepts_for_post(nil)).to eq([])
+    end
+
+    it "returns empty array when no existing concepts" do
+      allow(DiscourseAi::InferredConcepts::Manager).to receive(:list_concepts).and_return([])
+
+      result = described_class.match_existing_concepts_for_post(post)
+      expect(result).to eq([])
+    end
+
+    it "matches concepts and applies them to post" do
+      expect(described_class).to receive(:post_content_for_analysis).with(post).and_return(
+        "content about testing",
+      )
+
+      expect(described_class).to receive(:match_concepts_to_content).with(
+        "content about testing",
+        %w[programming testing ruby],
+      ).and_return(["testing"])
+
+      expect(InferredConcept).to receive(:where).with(name: ["testing"]).and_return([concept2])
+
+      expect(described_class).to receive(:apply_to_post).with(post, [concept2])
+
+      result = described_class.match_existing_concepts_for_post(post)
+      expect(result).to eq([concept2])
+    end
+  end
+
+  describe ".match_concepts_to_content" do
+    it "returns empty array for blank content or concept list" do
+      expect(described_class.match_concepts_to_content("", ["concept1"])).to eq([])
+      expect(described_class.match_concepts_to_content(nil, ["concept1"])).to eq([])
+      expect(described_class.match_concepts_to_content("content", [])).to eq([])
+      expect(described_class.match_concepts_to_content("content", nil)).to eq([])
+    end
+
+    it "uses ConceptMatcher persona to match concepts" do
+      content = "This is about Ruby programming"
+      concept_list = %w[programming testing ruby]
+      expected_response = [['{"matching_concepts": ["programming", "ruby"]}']]
+
+      persona_class_double = double("ConceptMatcherClass")
+      persona_double = double("ConceptMatcher")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(
+        persona_class_double,
+      )
+      expect(persona_class_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:class).and_return(persona_class_double)
+      expect(persona_class_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(expected_response)
+
+      result = described_class.match_concepts_to_content(content, concept_list)
+      expect(result).to eq(%w[programming ruby])
+    end
+
+    it "handles invalid JSON response gracefully" do
+      content = "Test content"
+      concept_list = ["concept1"]
+      invalid_response = [["invalid json"]]
+
+      persona_class_double = double("ConceptMatcherClass")
+      persona_double = double("ConceptMatcher")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(
+        persona_class_double,
+      )
+      expect(persona_class_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:class).and_return(persona_class_double)
+      expect(persona_class_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(invalid_response)
+
+      expect { described_class.match_concepts_to_content(content, concept_list) }.to raise_error(
+        JSON::ParserError,
+      )
+    end
+
+    it "returns empty array when no matching concepts found" do
+      content = "This is about something else"
+      concept_list = %w[programming testing]
+      expected_response = [['{"matching_concepts": []}']]
+
+      persona_class_double = double("ConceptMatcherClass")
+      persona_double = double("ConceptMatcher")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(
+        persona_class_double,
+      )
+      expect(persona_class_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:class).and_return(persona_class_double)
+      expect(persona_class_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(expected_response)
+
+      result = described_class.match_concepts_to_content(content, concept_list)
+      expect(result).to eq([])
+    end
+
+    it "handles missing matching_concepts key in response" do
+      content = "Test content"
+      concept_list = ["concept1"]
+      expected_response = [['{"other_key": ["value"]}']]
+
+      persona_class_double = double("ConceptMatcherClass")
+      persona_double = double("ConceptMatcher")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(
+        persona_class_double,
+      )
+      expect(persona_class_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:class).and_return(persona_class_double)
+      expect(persona_class_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(expected_response)
+
+      result = described_class.match_concepts_to_content(content, concept_list)
+      expect(result).to eq([])
+    end
+  end
+end

spec/lib/inferred_concepts/finder_spec.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::InferredConcepts::Finder do
+  fab!(:topic) { Fabricate(:topic, posts_count: 5, views: 200, like_count: 15) }
+  fab!(:post) { Fabricate(:post, like_count: 10) }
+  fab!(:concept1) { Fabricate(:inferred_concept, name: "programming") }
+  fab!(:concept2) { Fabricate(:inferred_concept, name: "testing") }
+  fab!(:llm_model) { Fabricate(:fake_model) }
+
+  before do
+    SiteSetting.inferred_concepts_generate_persona = -1
+    SiteSetting.inferred_concepts_deduplicate_persona = -1
+    SiteSetting.inferred_concepts_enabled = true
+  end
+
+  describe ".identify_concepts" do
+    it "returns empty array for blank content" do
+      expect(described_class.identify_concepts("")).to eq([])
+      expect(described_class.identify_concepts(nil)).to eq([])
+    end
+
+    it "uses ConceptFinder persona to identify concepts" do
+      content = "This is about Ruby programming and testing"
+      expected_response = [
+        { "type" => "text", "content" => '{"concepts": ["ruby", "programming", "testing"]}' },
+      ]
+
+      # Mock the persona and bot interaction
+      persona_double = double("ConceptFinder")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(persona_double)
+      expect(persona_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).with(llm_model.id).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(expected_response)
+
+      result = described_class.identify_concepts(content)
+      expect(result).to eq(%w[ruby programming testing])
+    end
+
+    it "handles invalid JSON response gracefully" do
+      content = "Test content"
+      invalid_response = [{ "type" => "text", "content" => "invalid json" }]
+
+      persona_double = double("ConceptFinder")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(persona_double)
+      expect(persona_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).with(llm_model.id).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(invalid_response)
+
+      expect { described_class.identify_concepts(content) }.to raise_error(JSON::ParserError)
+    end
+  end
+
+  describe ".create_or_find_concepts" do
+    it "returns empty array for blank concept names" do
+      expect(described_class.create_or_find_concepts([])).to eq([])
+      expect(described_class.create_or_find_concepts(nil)).to eq([])
+    end
+
+    it "creates new concepts for new names" do
+      concept_names = %w[new_concept1 new_concept2]
+      result = described_class.create_or_find_concepts(concept_names)
+
+      expect(result.length).to eq(2)
+      expect(result.map(&:name)).to match_array(concept_names)
+      expect(InferredConcept.where(name: concept_names).count).to eq(2)
+    end
+
+    it "finds existing concepts" do
+      concept_names = %w[programming testing]
+      result = described_class.create_or_find_concepts(concept_names)
+
+      expect(result.length).to eq(2)
+      expect(result).to include(concept1, concept2)
+    end
+
+    it "handles mix of new and existing concepts" do
+      concept_names = %w[programming new_concept]
+      result = described_class.create_or_find_concepts(concept_names)
+
+      expect(result.length).to eq(2)
+      expect(result.map(&:name)).to match_array(concept_names)
+    end
+  end
+
+  describe ".find_candidate_topics" do
+    let!(:good_topic) { Fabricate(:topic, posts_count: 6, views: 150, like_count: 12) }
+    let!(:bad_topic) { Fabricate(:topic, posts_count: 2, views: 50, like_count: 2) }
+    let!(:topic_with_concepts) do
+      t = Fabricate(:topic, posts_count: 8, views: 200, like_count: 20)
+      t.inferred_concepts << concept1
+      t
+    end
+
+    it "finds topics meeting minimum criteria" do
+      candidates =
+        described_class.find_candidate_topics(min_posts: 5, min_views: 100, min_likes: 10)
+
+      expect(candidates).to include(good_topic)
+      expect(candidates).not_to include(bad_topic)
+      expect(candidates).not_to include(topic_with_concepts) # already has concepts
+    end
+
+    it "respects limit parameter" do
+      candidates = described_class.find_candidate_topics(limit: 1)
+      expect(candidates.length).to be <= 1
+    end
+
+    it "excludes specified topic IDs" do
+      candidates = described_class.find_candidate_topics(exclude_topic_ids: [good_topic.id])
+      expect(candidates).not_to include(good_topic)
+    end
+
+    it "filters by category IDs when provided" do
+      category = Fabricate(:category)
+      topic_in_category =
+        Fabricate(:topic, category: category, posts_count: 6, views: 150, like_count: 12)
+
+      candidates = described_class.find_candidate_topics(category_ids: [category.id])
+
+      expect(candidates).to include(topic_in_category)
+      expect(candidates).not_to include(good_topic)
+    end
+
+    it "filters by creation date" do
+      old_topic =
+        Fabricate(:topic, posts_count: 6, views: 150, like_count: 12, created_at: 45.days.ago)
+
+      candidates = described_class.find_candidate_topics(created_after: 30.days.ago)
+
+      expect(candidates).to include(good_topic)
+      expect(candidates).not_to include(old_topic)
+    end
+  end
+
+  describe ".find_candidate_posts" do
+    let!(:good_post) { Fabricate(:post, like_count: 8, post_number: 2) }
+    let!(:bad_post) { Fabricate(:post, like_count: 2, post_number: 2) }
+    let!(:first_post) { Fabricate(:post, like_count: 10, post_number: 1) }
+    let!(:post_with_concepts) do
+      p = Fabricate(:post, like_count: 15, post_number: 3)
+      p.inferred_concepts << concept1
+      p
+    end
+
+    it "finds posts meeting minimum criteria" do
+      candidates = described_class.find_candidate_posts(min_likes: 5)
+
+      expect(candidates).to include(good_post)
+      expect(candidates).not_to include(bad_post)
+      expect(candidates).not_to include(post_with_concepts) # already has concepts
+    end
+
+    it "excludes first posts by default" do
+      candidates = described_class.find_candidate_posts(min_likes: 5)
+
+      expect(candidates).not_to include(first_post)
+    end
+
+    it "can include first posts when specified" do
+      candidates = described_class.find_candidate_posts(min_likes: 5, exclude_first_posts: false)
+
+      expect(candidates).to include(first_post)
+    end
+
+    it "respects limit parameter" do
+      candidates = described_class.find_candidate_posts(limit: 1)
+      expect(candidates.length).to be <= 1
+    end
+
+    it "excludes specified post IDs" do
+      candidates = described_class.find_candidate_posts(exclude_post_ids: [good_post.id])
+      expect(candidates).not_to include(good_post)
+    end
+
+    it "filters by category IDs when provided" do
+      category = Fabricate(:category)
+      topic_in_category = Fabricate(:topic, category: category)
+      post_in_category = Fabricate(:post, topic: topic_in_category, like_count: 8, post_number: 2)
+
+      candidates = described_class.find_candidate_posts(category_ids: [category.id])
+
+      expect(candidates).to include(post_in_category)
+      expect(candidates).not_to include(good_post)
+    end
+
+    it "filters by creation date" do
+      old_post = Fabricate(:post, like_count: 8, post_number: 2, created_at: 45.days.ago)
+
+      candidates = described_class.find_candidate_posts(created_after: 30.days.ago)
+
+      expect(candidates).to include(good_post)
+      expect(candidates).not_to include(old_post)
+    end
+  end
+
+  describe ".deduplicate_concepts" do
+    it "returns empty result for blank concept names" do
+      result = described_class.deduplicate_concepts([])
+      expect(result).to eq({ deduplicated_concepts: [], mapping: {} })
+
+      result = described_class.deduplicate_concepts(nil)
+      expect(result).to eq({ deduplicated_concepts: [], mapping: {} })
+    end
+
+    it "uses ConceptDeduplicator persona to deduplicate concepts" do
+      concept_names = ["ruby", "Ruby programming", "testing", "unit testing"]
+      expected_response = [
+        { "type" => "text", "content" => '{"streamlined_tags": ["ruby", "testing"]}' },
+      ]
+
+      persona_double = double("ConceptDeduplicator")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(persona_double)
+      expect(persona_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).with(llm_model.id).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(expected_response)
+
+      result = described_class.deduplicate_concepts(concept_names)
+      expect(result).to eq(%w[ruby testing])
+    end
+
+    it "handles invalid JSON response gracefully" do
+      concept_names = %w[concept1 concept2]
+      invalid_response = [{ "type" => "text", "content" => "invalid json" }]
+
+      persona_double = double("ConceptDeduplicator")
+      bot_double = double("Bot")
+
+      expect(AiPersona).to receive_message_chain(:all_personas, :find).and_return(persona_double)
+      expect(persona_double).to receive(:new).and_return(persona_double)
+      expect(persona_double).to receive(:default_llm_id).and_return(llm_model.id)
+      expect(LlmModel).to receive(:find).with(llm_model.id).and_return(llm_model)
+      expect(DiscourseAi::Personas::Bot).to receive(:as).and_return(bot_double)
+      expect(bot_double).to receive(:reply).and_return(invalid_response)
+
+      expect { described_class.deduplicate_concepts(concept_names) }.to raise_error(
+        JSON::ParserError,
+      )
+    end
+  end
+end

spec/lib/inferred_concepts/manager_spec.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::InferredConcepts::Manager do
+  fab!(:topic) { Fabricate(:topic) }
+  fab!(:post) { Fabricate(:post) }
+  fab!(:concept1) { Fabricate(:inferred_concept, name: "programming") }
+  fab!(:concept2) { Fabricate(:inferred_concept, name: "testing") }
+
+  describe ".list_concepts" do
+    it "returns all concepts sorted by name" do
+      concepts = described_class.list_concepts
+      expect(concepts).to include("programming", "testing")
+      expect(concepts).to eq(concepts.sort)
+    end
+
+    it "respects limit parameter" do
+      concepts = described_class.list_concepts(limit: 1)
+      expect(concepts.length).to eq(1)
+    end
+
+    it "returns empty array when no concepts exist" do
+      InferredConcept.destroy_all
+      concepts = described_class.list_concepts
+      expect(concepts).to eq([])
+    end
+  end
+
+  describe ".generate_concepts_from_content" do
+    before do
+      SiteSetting.inferred_concepts_generate_persona = -1
+      SiteSetting.inferred_concepts_enabled = true
+    end
+
+    it "returns empty array for blank content" do
+      expect(described_class.generate_concepts_from_content("")).to eq([])
+      expect(described_class.generate_concepts_from_content(nil)).to eq([])
+    end
+
+    it "delegates to Finder.identify_concepts" do
+      content = "This is about Ruby programming"
+      expect(DiscourseAi::InferredConcepts::Finder).to receive(:identify_concepts).with(
+        content,
+      ).and_return(%w[ruby programming])
+
+      expect(DiscourseAi::InferredConcepts::Finder).to receive(:create_or_find_concepts).with(
+        %w[ruby programming],
+      ).and_return([concept1])
+
+      result = described_class.generate_concepts_from_content(content)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".generate_concepts_from_topic" do
+    it "returns empty array for blank topic" do
+      expect(described_class.generate_concepts_from_topic(nil)).to eq([])
+    end
+
+    it "extracts content and generates concepts" do
+      expect(DiscourseAi::InferredConcepts::Applier).to receive(:topic_content_for_analysis).with(
+        topic,
+      ).and_return("topic content")
+
+      expect(described_class).to receive(:generate_concepts_from_content).with(
+        "topic content",
+      ).and_return([concept1])
+
+      result = described_class.generate_concepts_from_topic(topic)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".generate_concepts_from_post" do
+    it "returns empty array for blank post" do
+      expect(described_class.generate_concepts_from_post(nil)).to eq([])
+    end
+
+    it "extracts content and generates concepts" do
+      expect(DiscourseAi::InferredConcepts::Applier).to receive(:post_content_for_analysis).with(
+        post,
+      ).and_return("post content")
+
+      expect(described_class).to receive(:generate_concepts_from_content).with(
+        "post content",
+      ).and_return([concept1])
+
+      result = described_class.generate_concepts_from_post(post)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".match_topic_to_concepts" do
+    it "returns empty array for blank topic" do
+      expect(described_class.match_topic_to_concepts(nil)).to eq([])
+    end
+
+    it "delegates to Applier.match_existing_concepts" do
+      expect(DiscourseAi::InferredConcepts::Applier).to receive(:match_existing_concepts).with(
+        topic,
+      ).and_return([concept1])
+
+      result = described_class.match_topic_to_concepts(topic)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".match_post_to_concepts" do
+    it "returns empty array for blank post" do
+      expect(described_class.match_post_to_concepts(nil)).to eq([])
+    end
+
+    it "delegates to Applier.match_existing_concepts_for_post" do
+      expect(DiscourseAi::InferredConcepts::Applier).to receive(
+        :match_existing_concepts_for_post,
+      ).with(post).and_return([concept1])
+
+      result = described_class.match_post_to_concepts(post)
+      expect(result).to eq([concept1])
+    end
+  end
+
+  describe ".search_topics_by_concept" do
+    it "returns empty array for non-existent concept" do
+      result = described_class.search_topics_by_concept("nonexistent")
+      expect(result).to eq([])
+    end
+
+    it "returns topics associated with concept" do
+      concept1.topics << topic
+      result = described_class.search_topics_by_concept("programming")
+      expect(result).to include(topic)
+    end
+  end
+
+  describe ".search_posts_by_concept" do
+    it "returns empty array for non-existent concept" do
+      result = described_class.search_posts_by_concept("nonexistent")
+      expect(result).to eq([])
+    end
+
+    it "returns posts associated with concept" do
+      concept1.posts << post
+      result = described_class.search_posts_by_concept("programming")
+      expect(result).to include(post)
+    end
+  end
+
+  describe ".match_content_to_concepts" do
+    it "returns empty array when no concepts exist" do
+      InferredConcept.destroy_all
+      result = described_class.match_content_to_concepts("some content")
+      expect(result).to eq([])
+    end
+
+    it "delegates to Applier.match_concepts_to_content" do
+      content = "programming content"
+      existing_concepts = %w[programming testing]
+
+      expect(InferredConcept).to receive_message_chain(:all, :pluck).with(:name).and_return(
+        existing_concepts,
+      )
+
+      expect(DiscourseAi::InferredConcepts::Applier).to receive(:match_concepts_to_content).with(
+        content,
+        existing_concepts,
+      ).and_return(["programming"])
+
+      result = described_class.match_content_to_concepts(content)
+      expect(result).to eq(["programming"])
+    end
+  end
+
+  describe ".find_candidate_topics" do
+    it "delegates to Finder.find_candidate_topics with options" do
+      opts = { limit: 50, min_posts: 3 }
+      expect(DiscourseAi::InferredConcepts::Finder).to receive(:find_candidate_topics).with(
+        **opts,
+      ).and_return([topic])
+
+      result = described_class.find_candidate_topics(opts)
+      expect(result).to eq([topic])
+    end
+  end
+
+  describe ".find_candidate_posts" do
+    it "delegates to Finder.find_candidate_posts with options" do
+      opts = { limit: 25, min_likes: 2 }
+      expect(DiscourseAi::InferredConcepts::Finder).to receive(:find_candidate_posts).with(
+        **opts,
+      ).and_return([post])
+
+      result = described_class.find_candidate_posts(opts)
+      expect(result).to eq([post])
+    end
+  end
+
+  describe ".deduplicate_concepts_by_letter" do
+    before do
+      # Create test concepts
+      %w[apple application banana berry cat car dog].each do |name|
+        Fabricate(:inferred_concept, name: name)
+      end
+    end
+
+    it "groups concepts by first letter and deduplicates" do
+      expect(DiscourseAi::InferredConcepts::Finder).to receive(:deduplicate_concepts).at_least(
+        :once,
+      ).and_return(%w[apple banana cat dog])
+
+      expect(InferredConcept).to receive(:where).and_call_original
+      expect(InferredConcept).to receive(:insert_all).and_call_original
+
+      described_class.deduplicate_concepts_by_letter
+    end
+
+    it "handles empty concept list" do
+      InferredConcept.destroy_all
+      expect { described_class.deduplicate_concepts_by_letter }.not_to raise_error
+    end
+  end
+end

spec/lib/personas/concept_deduplicator_spec.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Personas::ConceptDeduplicator do
+  let(:persona) { described_class.new }
+
+  describe ".default_enabled" do
+    it "is disabled by default" do
+      expect(described_class.default_enabled).to eq(false)
+    end
+  end
+
+  describe "#system_prompt" do
+    let(:prompt) { persona.system_prompt }
+
+    it "explains the deduplication task" do
+      expect(prompt).to include("streamline this list by merging entries who are similar")
+      expect(prompt).to include("machine-generated tags")
+    end
+
+    it "provides step-by-step instructions" do
+      expect(prompt).to include("1. Review the entire list")
+      expect(prompt).to include("2. Identify and remove any exact duplicates")
+      expect(prompt).to include("3. Look for tags that are too specific")
+      expect(prompt).to include("4. If there are multiple tags that convey similar concepts")
+      expect(prompt).to include("5. Ensure that the remaining tags are relevant")
+    end
+
+    it "defines criteria for best tags" do
+      expect(prompt).to include("Relevance: How well does the tag describe")
+      expect(prompt).to include("Generality: Is the tag specific enough")
+      expect(prompt).to include("Clarity: Is the tag easy to understand")
+      expect(prompt).to include("Popularity: Would this tag likely be used")
+    end
+
+    it "includes example input and output" do
+      expect(prompt).to include("Example Input:")
+      expect(prompt).to include("AI Bias, AI Bots, AI Ethics")
+      expect(prompt).to include("Example Output:")
+      expect(prompt).to include("AI, AJAX, API, APK")
+    end
+
+    it "specifies output format" do
+      expect(prompt).to include("<streamlined_tags>")
+      expect(prompt).to include("<o>")
+      expect(prompt).to include('"streamlined_tags": ["tag1", "tag3"]')
+      expect(prompt).to include("</o>")
+    end
+
+    it "emphasizes maintaining essence" do
+      expect(prompt).to include("maintaining the essence of the original list")
+    end
+  end
+
+  describe "#response_format" do
+    it "defines correct response format" do
+      format = persona.response_format
+
+      expect(format).to eq([{ "key" => "streamlined_tags", "type" => "array" }])
+    end
+  end
+
+  describe "deduplication guidelines" do
+    let(:prompt) { persona.system_prompt }
+
+    it "addresses duplicate removal" do
+      expect(prompt).to include("exact duplicates")
+    end
+
+    it "addresses specificity concerns" do
+      expect(prompt).to include("too specific or niche")
+      expect(prompt).to include("more general terms")
+    end
+
+    it "addresses similar concept merging" do
+      expect(prompt).to include("similar concepts, choose the best one")
+    end
+
+    it "emphasizes relevance and utility" do
+      expect(prompt).to include("relevant and useful for describing the content")
+    end
+  end
+end

spec/lib/personas/concept_finder_spec.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Personas::ConceptFinder do
+  let(:persona) { described_class.new }
+
+  describe ".default_enabled" do
+    it "is disabled by default" do
+      expect(described_class.default_enabled).to eq(false)
+    end
+  end
+
+  describe "#system_prompt" do
+    before do
+      Fabricate(:inferred_concept, name: "programming")
+      Fabricate(:inferred_concept, name: "testing")
+      Fabricate(:inferred_concept, name: "ruby")
+    end
+
+    it "includes guidelines for concept extraction" do
+      prompt = persona.system_prompt
+
+      expect(prompt).to include("advanced concept tagging system")
+      expect(prompt).to include("Extract up to 7 concepts")
+      expect(prompt).to include("single words or short phrases")
+      expect(prompt).to include("substantive topics, themes, technologies")
+      expect(prompt).to include("JSON object")
+      expect(prompt).to include('"concepts"')
+    end
+
+    it "includes existing concepts when available" do
+      prompt = persona.system_prompt
+
+      expect(prompt).to include("following concepts already exist")
+      expect(prompt).to include("programming")
+      expect(prompt).to include("testing")
+      expect(prompt).to include("ruby")
+      expect(prompt).to include("reuse these existing concepts")
+    end
+
+    it "handles empty existing concepts" do
+      InferredConcept.destroy_all
+      prompt = persona.system_prompt
+
+      expect(prompt).not_to include("following concepts already exist")
+      expect(prompt).to include("advanced concept tagging system")
+    end
+
+    it "limits existing concepts to 100" do
+      expect(DiscourseAi::InferredConcepts::Manager).to receive(:list_concepts).with(
+        limit: 100,
+      ).and_return(%w[concept1 concept2])
+
+      persona.system_prompt
+    end
+
+    it "includes format instructions" do
+      prompt = persona.system_prompt
+
+      expect(prompt).to include("<o>")
+      expect(prompt).to include('{"concepts": ["concept1", "concept2", "concept3"]}')
+      expect(prompt).to include("</o>")
+    end
+
+    it "includes language preservation instruction" do
+      prompt = persona.system_prompt
+
+      expect(prompt).to include("original language of the text")
+    end
+  end
+
+  describe "#response_format" do
+    it "defines correct response format" do
+      format = persona.response_format
+
+      expect(format).to eq(
+        [{ "key" => "concepts", "type" => "array", "items" => { "type" => "string" } }],
+      )
+    end
+  end
+
+  describe "prompt quality guidelines" do
+    let(:prompt) { persona.system_prompt }
+
+    it "emphasizes avoiding generic terms" do
+      expect(prompt).to include('Avoid overly general terms like "discussion" or "question"')
+    end
+
+    it "focuses on substantive content" do
+      expect(prompt).to include("substantive topics, themes, technologies, methodologies")
+    end
+
+    it "limits concept length" do
+      expect(prompt).to include("1-3 words maximum")
+    end
+
+    it "emphasizes core content relevance" do
+      expect(prompt).to include("relevant to the core content")
+    end
+
+    it "discourages proper nouns unless they're technologies" do
+      expect(prompt).to include(
+        "Do not include proper nouns unless they represent key technologies",
+      )
+    end
+  end
+end

spec/lib/personas/concept_matcher_spec.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Personas::ConceptMatcher do
+  let(:persona) { described_class.new }
+
+  describe ".default_enabled" do
+    it "is disabled by default" do
+      expect(described_class.default_enabled).to eq(false)
+    end
+  end
+
+  describe "#system_prompt" do
+    let(:prompt) { persona.system_prompt }
+
+    it "explains the concept matching task" do
+      expect(prompt).to include("advanced concept matching system")
+      expect(prompt).to include("determines which concepts from a provided list are relevant")
+    end
+
+    it "provides matching guidelines" do
+      expect(prompt).to include("Only select concepts that are clearly relevant")
+      expect(prompt).to include("content must substantially discuss or relate")
+      expect(prompt).to include("Superficial mentions are not enough")
+      expect(prompt).to include("Be precise and selective")
+      expect(prompt).to include("Consider both explicit mentions and implicit discussions")
+    end
+
+    it "emphasizes exact concept matching" do
+      expect(prompt).to include("Only select from the exact concepts in the provided list")
+      expect(prompt).to include("do not add new concepts")
+      expect(prompt).to include("If no concepts from the list match")
+    end
+
+    it "includes placeholder for concept list" do
+      expect(prompt).to include("{inferred_concepts}")
+      expect(prompt).to include("The list of available concepts is:")
+    end
+
+    it "specifies output format" do
+      expect(prompt).to include("matching_concepts")
+      expect(prompt).to include("<o>")
+      expect(prompt).to include('{"matching_concepts": ["concept1", "concept3", "concept5"]}')
+      expect(prompt).to include("</o>")
+    end
+
+    it "emphasizes language preservation" do
+      expect(prompt).to include("original language of the text")
+    end
+
+    it "handles empty matches" do
+      expect(prompt).to include("return an empty array")
+    end
+  end
+
+  describe "#response_format" do
+    it "defines correct response format" do
+      format = persona.response_format
+
+      expect(format).to eq([{ "key" => "matching_concepts", "type" => "array" }])
+    end
+  end
+
+  describe "matching criteria" do
+    let(:prompt) { persona.system_prompt }
+
+    it "requires substantial discussion" do
+      expect(prompt).to include("substantially discuss or relate to the concept")
+    end
+
+    it "rejects superficial mentions" do
+      expect(prompt).to include("Superficial mentions are not enough")
+    end
+
+    it "emphasizes precision" do
+      expect(prompt).to include("precise and selective")
+      expect(prompt).to include("tangentially related")
+    end
+
+    it "considers implicit discussions" do
+      expect(prompt).to include("explicit mentions and implicit discussions")
+    end
+
+    it "restricts to provided list only" do
+      expect(prompt).to include("exact concepts in the provided list")
+      expect(prompt).to include("do not add new concepts")
+    end
+  end
+end

spec/models/inferred_concept_spec.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+RSpec.describe InferredConcept do
+  describe "validations" do
+    it "requires a name" do
+      concept = InferredConcept.new
+      expect(concept).not_to be_valid
+      expect(concept.errors[:name]).to include("can't be blank")
+    end
+
+    it "requires unique names" do
+      Fabricate(:inferred_concept, name: "ruby")
+      concept = InferredConcept.new(name: "ruby")
+      expect(concept).not_to be_valid
+      expect(concept.errors[:name]).to include("has already been taken")
+    end
+
+    it "is valid with a unique name" do
+      concept = Fabricate(:inferred_concept, name: "programming")
+      expect(concept).to be_valid
+    end
+  end
+
+  describe "associations" do
+    fab!(:topic) { Fabricate(:topic) }
+    fab!(:post) { Fabricate(:post) }
+    fab!(:concept) { Fabricate(:inferred_concept, name: "programming") }
+
+    it "can be associated with topics" do
+      concept.topics << topic
+      expect(concept.topics).to include(topic)
+      expect(topic.inferred_concepts).to include(concept)
+    end
+
+    it "can be associated with posts" do
+      concept.posts << post
+      expect(concept.posts).to include(post)
+      expect(post.inferred_concepts).to include(concept)
+    end
+
+    it "can have multiple topics and posts" do
+      topic2 = Fabricate(:topic)
+      post2 = Fabricate(:post)
+
+      concept.topics << [topic, topic2]
+      concept.posts << [post, post2]
+
+      expect(concept.topics.count).to eq(2)
+      expect(concept.posts.count).to eq(2)
+    end
+  end
+
+  describe "database constraints" do
+    it "has the expected schema" do
+      concept = Fabricate(:inferred_concept)
+      expect(concept).to respond_to(:name)
+      expect(concept).to respond_to(:created_at)
+      expect(concept).to respond_to(:updated_at)
+    end
+  end
+end