From e255c7a8f03685db5a5836e0332e9d28f8f0268c Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 6 Mar 2025 09:41:09 +1100 Subject: [PATCH] FEATURE: automation triage using personas (#1126) ## LLM Persona Triage - Allows automated responses to posts using AI personas - Configurable to respond as regular posts or whispers - Adds context-aware formatting for topics and private messages - Provides special handling for topic metadata (title, category, tags) ## LLM Tool Triage - Enables custom AI tools to process and respond to posts - Tools can analyze post content and invoke personas when needed - Zero-parameter tools can be used for automated workflows - Not enabled in production yet ## Implementation Details - Added new scriptable registration in discourse_automation/ directory - Created core implementation in lib/automation/ modules - Enhanced PromptMessagesBuilder with topic-style formatting - Added helper methods for persona and tool selection in UI - Extended AI Bot functionality to support whisper responses - Added rate limiting to prevent abuse ## Other Changes - Added comprehensive test coverage for both automation types - Enhanced tool runner with LLM integration capabilities - Improved error handling and logging This feature allows forum admins to configure AI personas to automatically respond to posts based on custom criteria and leverage AI tools for more complex triage workflows. Tool Triage has been disabled in production while we finalize details of new scripting capabilities. --- config/locales/client.en.yml | 18 ++ config/locales/server.en.yml | 6 + discourse_automation/llm_persona_triage.rb | 55 ++++ discourse_automation/llm_tool_triage.rb | 49 +++ lib/ai_bot/playground.rb | 17 +- lib/ai_bot/tool_runner.rb | 283 ++++++++++++++---- lib/automation.rb | 23 ++ lib/automation/llm_persona_triage.rb | 26 ++ lib/automation/llm_tool_triage.rb | 21 ++ lib/completions/prompt.rb | 15 + lib/completions/prompt_messages_builder.rb | 56 +++- plugin.rb | 2 + .../prompt_messages_builder_spec.rb | 24 ++ .../llm_persona_triage_spec.rb | 208 +++++++++++++ .../llm_tool_triage_spec.rb | 113 +++++++ spec/lib/modules/ai_bot/playground_spec.rb | 10 +- spec/models/ai_tool_spec.rb | 34 +++ 17 files changed, 899 insertions(+), 61 deletions(-) create mode 100644 discourse_automation/llm_persona_triage.rb create mode 100644 discourse_automation/llm_tool_triage.rb create mode 100644 lib/automation/llm_persona_triage.rb create mode 100644 lib/automation/llm_tool_triage.rb create mode 100644 spec/lib/discourse_automation/llm_persona_triage_spec.rb create mode 100644 spec/lib/discourse_automation/llm_tool_triage_spec.rb diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index d5647605..88bde180 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -90,6 +90,24 @@ en: label: "Top P" description: "Top P to use for the LLM, increase to increase randomness (leave empty to use model default)" + llm_tool_triage: + fields: + model: + label: "Model" + description: "The default language model used for triage" + tool: + label: "Tool" + description: "Tool to use for triage (tool must have no parameters defined)" + + + llm_persona_triage: + fields: + persona: + label: "Persona" + description: "AI Persona to use for triage (must have default LLM and User set)" + whisper: + label: "Reply as Whisper" + description: "Whether the persona's response should be a whisper" llm_triage: fields: system_prompt: diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index f116f978..13a3685d 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -6,6 +6,12 @@ en: spam: "Flag as spam and hide post" spam_silence: "Flag as spam, hide post and silence user" scriptables: + llm_tool_triage: + title: Triage posts using AI Tool + description: "Triage posts using custom logic in an AI tool" + llm_persona_triage: + title: Triage posts using AI Persona + description: "Respond to posts using a specific AI persona" llm_triage: title: Triage posts using AI description: "Triage posts using a large language model" diff --git a/discourse_automation/llm_persona_triage.rb b/discourse_automation/llm_persona_triage.rb new file mode 100644 index 00000000..b78e418c --- /dev/null +++ b/discourse_automation/llm_persona_triage.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +if defined?(DiscourseAutomation) + DiscourseAutomation::Scriptable.add("llm_persona_triage") do + version 1 + run_in_background + + triggerables %i[post_created_edited] + + field :persona, + component: :choices, + required: true, + extra: { + content: DiscourseAi::Automation.available_persona_choices, + } + field :whisper, component: :boolean + + script do |context, fields| + post = context["post"] + next if post&.user&.bot? + + persona_id = fields["persona"]["value"] + whisper = fields["whisper"]["value"] + + begin + RateLimiter.new( + Discourse.system_user, + "llm_persona_triage_#{post.id}", + SiteSetting.ai_automation_max_triage_per_post_per_minute, + 1.minute, + ).performed! + + RateLimiter.new( + Discourse.system_user, + "llm_persona_triage", + SiteSetting.ai_automation_max_triage_per_minute, + 1.minute, + ).performed! + + DiscourseAi::Automation::LlmPersonaTriage.handle( + post: post, + persona_id: persona_id, + whisper: whisper, + automation: self.automation, + ) + rescue => e + Discourse.warn_exception( + e, + message: "llm_persona_triage: skipped triage on post #{post.id}", + ) + raise e if Rails.env.tests? + end + end + end +end diff --git a/discourse_automation/llm_tool_triage.rb b/discourse_automation/llm_tool_triage.rb new file mode 100644 index 00000000..885f81fb --- /dev/null +++ b/discourse_automation/llm_tool_triage.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +# TODO: this is still highly experimental and subject to a lot of change +# leaving it off in production for now Sam +if defined?(DiscourseAutomation) && !Rails.env.production? + DiscourseAutomation::Scriptable.add("llm_tool_triage") do + version 1 + run_in_background + + triggerables %i[post_created_edited] + + field :tool, + component: :choices, + required: true, + extra: { + content: DiscourseAi::Automation.available_custom_tools, + } + + script do |context, fields| + tool_id = fields["tool"]["value"] + post = context["post"] + return if post&.user&.bot? + + begin + RateLimiter.new( + Discourse.system_user, + "llm_tool_triage_#{post.id}", + SiteSetting.ai_automation_max_triage_per_post_per_minute, + 1.minute, + ).performed! + + RateLimiter.new( + Discourse.system_user, + "llm_tool_triage", + SiteSetting.ai_automation_max_triage_per_minute, + 1.minute, + ).performed! + + DiscourseAi::Automation::LlmToolTriage.handle( + post: post, + tool_id: tool_id, + automation: self.automation, + ) + rescue => e + Discourse.warn_exception(e, message: "llm_tool_triage: skipped triage on post #{post.id}") + end + end + end +end diff --git a/lib/ai_bot/playground.rb b/lib/ai_bot/playground.rb index f9e2fd8e..2c9fee2c 100644 --- a/lib/ai_bot/playground.rb +++ b/lib/ai_bot/playground.rb @@ -170,7 +170,7 @@ module DiscourseAi schedule_bot_reply(post) if can_attach?(post) end - def conversation_context(post) + def conversation_context(post, style: nil) # Pay attention to the `post_number <= ?` here. # We want to inject the last post as context because they are translated differently. @@ -205,6 +205,7 @@ module DiscourseAi ) builder = DiscourseAi::Completions::PromptMessagesBuilder.new + builder.topic = post.topic context.reverse_each do |raw, username, custom_prompt, upload_ids| custom_prompt_translation = @@ -245,7 +246,7 @@ module DiscourseAi end end - builder.to_a + builder.to_a(style: style || (post.topic.private_message? ? :bot : :topic)) end def title_playground(post, user) @@ -418,7 +419,7 @@ module DiscourseAi result end - def reply_to(post, custom_instructions: nil, &blk) + def reply_to(post, custom_instructions: nil, whisper: nil, context_style: nil, &blk) # this is a multithreading issue # post custom prompt is needed and it may not # be properly loaded, ensure it is loaded @@ -428,12 +429,18 @@ module DiscourseAi post_streamer = nil post_type = - post.post_type == Post.types[:whisper] ? Post.types[:whisper] : Post.types[:regular] + ( + if (whisper || post.post_type == Post.types[:whisper]) + Post.types[:whisper] + else + Post.types[:regular] + end + ) context = get_context( participants: post.topic.allowed_users.map(&:username).join(", "), - conversation_context: conversation_context(post), + conversation_context: conversation_context(post, style: context_style), user: post.user, ) context[:post_id] = post.id diff --git a/lib/ai_bot/tool_runner.rb b/lib/ai_bot/tool_runner.rb index 17440ead..2a3899ef 100644 --- a/lib/ai_bot/tool_runner.rb +++ b/lib/ai_bot/tool_runner.rb @@ -38,6 +38,7 @@ module DiscourseAi attach_index(ctx) attach_upload(ctx) attach_chain(ctx) + attach_discourse(ctx) ctx.eval(framework_script) ctx end @@ -56,6 +57,7 @@ module DiscourseAi const llm = { truncate: _llm_truncate, + generate: _llm_generate, }; const index = { @@ -70,6 +72,24 @@ module DiscourseAi setCustomRaw: _chain_set_custom_raw, }; + const discourse = { + getPost: _discourse_get_post, + getUser: _discourse_get_user, + getPersona: function(name) { + return { + respondTo: function(params) { + result = _discourse_respond_to_persona(name, params); + if (result.error) { + throw new Error(result.error); + } + return result; + }, + }; + }, + }; + + const context = #{JSON.generate(@context)}; + function details() { return ""; }; JS end @@ -175,20 +195,60 @@ module DiscourseAi "_llm_truncate", ->(text, length) { @llm.tokenizer.truncate(text, length) }, ) + + mini_racer_context.attach( + "_llm_generate", + ->(prompt) do + in_attached_function do + @llm.generate( + convert_js_prompt_to_ruby(prompt), + user: llm_user, + feature_name: "custom_tool_#{tool.name}", + ) + end + end, + ) + end + + def convert_js_prompt_to_ruby(prompt) + if prompt.is_a?(String) + prompt + elsif prompt.is_a?(Hash) + messages = prompt["messages"] + if messages.blank? || !messages.is_a?(Array) + raise Discourse::InvalidParameters.new("Prompt must have messages") + end + messages.each(&:symbolize_keys!) + messages.each { |message| message[:type] = message[:type].to_sym } + DiscourseAi::Completions::Prompt.new(messages: prompt["messages"]) + else + raise Discourse::InvalidParameters.new("Prompt must be a string or a hash") + end + end + + def llm_user + @llm_user ||= + begin + @context[:llm_user] || post&.user || @bot_user + end + end + + def post + return @post if defined?(@post) + post_id = @context[:post_id] + @post = post_id && Post.find_by(id: post_id) end def attach_index(mini_racer_context) mini_racer_context.attach( "_index_search", ->(*params) do - begin + in_attached_function do query, options = params self.running_attached_function = true options ||= {} options = options.symbolize_keys self.rag_search(query, **options) - ensure - self.running_attached_function = false end end, ) @@ -198,31 +258,115 @@ module DiscourseAi mini_racer_context.attach("_chain_set_custom_raw", ->(raw) { self.custom_raw = raw }) end + def attach_discourse(mini_racer_context) + mini_racer_context.attach( + "_discourse_get_post", + ->(post_id) do + in_attached_function do + post = Post.find_by(id: post_id) + return nil if post.nil? + guardian = Guardian.new(Discourse.system_user) + recursive_as_json(PostSerializer.new(post, scope: guardian, root: false)) + end + end, + ) + + mini_racer_context.attach( + "_discourse_get_user", + ->(user_id_or_username) do + in_attached_function do + user = nil + + if user_id_or_username.is_a?(Integer) || + user_id_or_username.to_i.to_s == user_id_or_username + user = User.find_by(id: user_id_or_username.to_i) + else + user = User.find_by(username: user_id_or_username) + end + + return nil if user.nil? + + guardian = Guardian.new(Discourse.system_user) + recursive_as_json(UserSerializer.new(user, scope: guardian, root: false)) + end + end, + ) + + mini_racer_context.attach( + "_discourse_respond_to_persona", + ->(persona_name, params) do + in_attached_function do + # if we have 1000s of personas this can be slow ... we may need to optimize + persona_class = AiPersona.all_personas.find { |persona| persona.name == persona_name } + return { error: "Persona not found" } if persona_class.nil? + + persona = persona_class.new + bot = DiscourseAi::AiBot::Bot.as(@bot_user || persona.user, persona: persona) + playground = DiscourseAi::AiBot::Playground.new(bot) + + if @context[:post_id] + post = Post.find_by(id: @context[:post_id]) + return { error: "Post not found" } if post.nil? + + reply_post = + playground.reply_to( + post, + custom_instructions: params["instructions"], + whisper: params["whisper"], + ) + + if reply_post + return( + { success: true, post_id: reply_post.id, post_number: reply_post.post_number } + ) + else + return { error: "Failed to create reply" } + end + elsif @context[:message_id] && @context[:channel_id] + message = Chat::Message.find_by(id: @context[:message_id]) + channel = Chat::Channel.find_by(id: @context[:channel_id]) + return { error: "Message or channel not found" } if message.nil? || channel.nil? + + reply = + playground.reply_to_chat_message(message, channel, @context[:context_post_ids]) + + if reply + return { success: true, message_id: reply.id } + else + return { error: "Failed to create chat reply" } + end + else + return { error: "No valid context for response" } + end + end + end, + ) + end + def attach_upload(mini_racer_context) mini_racer_context.attach( "_upload_create", ->(filename, base_64_content) do begin - self.running_attached_function = true - # protect against misuse - filename = File.basename(filename) + in_attached_function do + # protect against misuse + filename = File.basename(filename) - Tempfile.create(filename) do |file| - file.binmode - file.write(Base64.decode64(base_64_content)) - file.rewind + Tempfile.create(filename) do |file| + file.binmode + file.write(Base64.decode64(base_64_content)) + file.rewind - upload = - UploadCreator.new( - file, - filename, - for_private_message: @context[:private_message], - ).create_for(@bot_user.id) + upload = + UploadCreator.new( + file, + filename, + for_private_message: @context[:private_message], + ).create_for(@bot_user.id) - { id: upload.id, short_url: upload.short_url, url: upload.url } + { id: upload.id, short_url: upload.short_url, url: upload.url } + end end - ensure - self.running_attached_function = false end end, ) @@ -238,18 +382,20 @@ module DiscourseAi raise TooManyRequestsError.new("Tool made too many HTTP requests") end - self.running_attached_function = true - headers = (options && options["headers"]) || {} + in_attached_function do + headers = (options && options["headers"]) || {} - result = {} - DiscourseAi::AiBot::Tools::Tool.send_http_request(url, headers: headers) do |response| - result[:body] = response.body - result[:status] = response.code.to_i + result = {} + DiscourseAi::AiBot::Tools::Tool.send_http_request( + url, + headers: headers, + ) do |response| + result[:body] = response.body + result[:status] = response.code.to_i + end + + result end - - result - ensure - self.running_attached_function = false end end, ) @@ -264,35 +410,70 @@ module DiscourseAi raise TooManyRequestsError.new("Tool made too many HTTP requests") end - self.running_attached_function = true - headers = (options && options["headers"]) || {} - body = options && options["body"] + in_attached_function do + headers = (options && options["headers"]) || {} + body = options && options["body"] - result = {} - DiscourseAi::AiBot::Tools::Tool.send_http_request( - url, - method: method, - headers: headers, - body: body, - ) do |response| - result[:body] = response.body - result[:status] = response.code.to_i + result = {} + DiscourseAi::AiBot::Tools::Tool.send_http_request( + url, + method: method, + headers: headers, + body: body, + ) do |response| + result[:body] = response.body + result[:status] = response.code.to_i + end + + result + rescue => e + if Rails.env.development? + p url + p options + p e + puts e.backtrace + end + raise e end - - result - rescue => e - p url - p options - p e - puts e.backtrace - raise e - ensure - self.running_attached_function = false end end, ) end end + + def in_attached_function + self.running_attached_function = true + yield + ensure + self.running_attached_function = false + end + + def recursive_as_json(obj) + case obj + when Array + obj.map { |item| recursive_as_json(item) } + when Hash + obj.transform_values { |value| recursive_as_json(value) } + when ActiveModel::Serializer, ActiveModel::ArraySerializer + recursive_as_json(obj.as_json) + when ActiveRecord::Base + recursive_as_json(obj.as_json) + else + # Handle objects that respond to as_json but aren't handled above + if obj.respond_to?(:as_json) + result = obj.as_json + if result.equal?(obj) + # If as_json returned the same object, return it to avoid infinite recursion + result + else + recursive_as_json(result) + end + else + # Primitive values like strings, numbers, booleans, nil + obj + end + end + end end end end diff --git a/lib/automation.rb b/lib/automation.rb index d40b2d12..f080234e 100644 --- a/lib/automation.rb +++ b/lib/automation.rb @@ -12,6 +12,15 @@ module DiscourseAi }, ] end + + def self.available_custom_tools + AiTool + .where(enabled: true) + .where("parameters = '[]'::jsonb") + .pluck(:id, :name, :description) + .map { |id, name, description| { id: id, translated_name: name, description: description } } + end + def self.available_models values = DB.query_hash(<<~SQL) SELECT display_name AS translated_name, id AS id @@ -28,5 +37,19 @@ module DiscourseAi values end + + def self.available_persona_choices + AiPersona + .joins(:user) + .where.not(user_id: nil) + .where.not(default_llm: nil) + .map do |persona| + { + id: persona.id, + translated_name: persona.name, + description: "#{persona.name} (#{persona.user.username})", + } + end + end end end diff --git a/lib/automation/llm_persona_triage.rb b/lib/automation/llm_persona_triage.rb new file mode 100644 index 00000000..753d2632 --- /dev/null +++ b/lib/automation/llm_persona_triage.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +module DiscourseAi + module Automation + module LlmPersonaTriage + def self.handle(post:, persona_id:, whisper: false, automation: nil) + ai_persona = AiPersona.find_by(id: persona_id) + return if ai_persona.nil? + + persona_class = ai_persona.class_instance + persona = persona_class.new + + bot_user = ai_persona.user + return if bot_user.nil? + + bot = DiscourseAi::AiBot::Bot.as(bot_user, persona: persona) + playground = DiscourseAi::AiBot::Playground.new(bot) + + playground.reply_to(post, whisper: whisper, context_style: :topic) + rescue => e + Rails.logger.error("Error in LlmPersonaTriage: #{e.message}\n#{e.backtrace.join("\n")}") + raise e if Rails.env.test? + nil + end + end + end +end diff --git a/lib/automation/llm_tool_triage.rb b/lib/automation/llm_tool_triage.rb new file mode 100644 index 00000000..978119a2 --- /dev/null +++ b/lib/automation/llm_tool_triage.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +module DiscourseAi + module Automation + module LlmToolTriage + def self.handle(post:, tool_id:, automation: nil) + tool = AiTool.find_by(id: tool_id) + return if !tool + return if !tool.parameters.blank? + + context = { + post_id: post.id, + automation_id: automation&.id, + automation_name: automation&.name, + } + + runner = tool.runner({}, llm: nil, bot_user: Discourse.system_user, context: context) + runner.invoke + end + end + end +end diff --git a/lib/completions/prompt.rb b/lib/completions/prompt.rb index 6afbc52b..b4f836b1 100644 --- a/lib/completions/prompt.rb +++ b/lib/completions/prompt.rb @@ -79,6 +79,21 @@ module DiscourseAi UploadEncoder.encode(upload_ids: message[:upload_ids], max_pixels: max_pixels) end + def ==(other) + return false unless other.is_a?(Prompt) + messages == other.messages && tools == other.tools && topic_id == other.topic_id && + post_id == other.post_id && max_pixels == other.max_pixels && + tool_choice == other.tool_choice + end + + def eql?(other) + self == other + end + + def hash + [messages, tools, topic_id, post_id, max_pixels, tool_choice].hash + end + private def validate_message(message) diff --git a/lib/completions/prompt_messages_builder.rb b/lib/completions/prompt_messages_builder.rb index 045e0e89..26fb24fa 100644 --- a/lib/completions/prompt_messages_builder.rb +++ b/lib/completions/prompt_messages_builder.rb @@ -4,8 +4,10 @@ module DiscourseAi module Completions class PromptMessagesBuilder MAX_CHAT_UPLOADS = 5 + MAX_TOPIC_UPLOADS = 5 attr_reader :chat_context_posts attr_reader :chat_context_post_upload_ids + attr_accessor :topic def initialize @raw_messages = [] @@ -41,6 +43,7 @@ module DiscourseAi def to_a(limit: nil, style: nil) return chat_array(limit: limit) if style == :chat + return topic_array if style == :topic result = [] # this will create a "valid" messages array @@ -127,6 +130,57 @@ module DiscourseAi private + def topic_array + raw_messages = @raw_messages.dup + user_content = +"You are operating in a Discourse forum.\n\n" + + if @topic + if @topic.private_message? + user_content << "Private message info.\n" + else + user_content << "Topic information:\n" + end + + user_content << "- URL: #{@topic.url}\n" + user_content << "- Title: #{@topic.title}\n" + if SiteSetting.tagging_enabled + tags = @topic.tags.pluck(:name) + tags -= DiscourseTagging.hidden_tag_names if tags.present? + user_content << "- Tags: #{tags.join(", ")}\n" if tags.present? + end + if !@topic.private_message? + user_content << "- Category: #{@topic.category.name}\n" if @topic.category + end + user_content << "- Number of replies: #{@topic.posts_count - 1}\n\n" + end + + last_user_message = raw_messages.pop + + upload_ids = [] + if raw_messages.present? + user_content << "Here is the conversation so far:\n" + raw_messages.each do |message| + user_content << "#{message[:name] || "User"}: #{message[:content]}\n" + upload_ids.concat(message[:upload_ids]) if message[:upload_ids].present? + end + end + + if last_user_message + user_content << "You are responding to #{last_user_message[:name] || "User"} who just said:\n #{last_user_message[:content]}" + if last_user_message[:upload_ids].present? + upload_ids.concat(last_user_message[:upload_ids]) + end + end + + user_message = { type: :user, content: user_content } + + if upload_ids.present? + user_message[:upload_ids] = upload_ids[-MAX_TOPIC_UPLOADS..-1] || upload_ids + end + + [user_message] + end + def chat_array(limit:) if @raw_messages.length > 1 buffer = @@ -155,7 +209,7 @@ module DiscourseAi end last_message = @raw_messages[-1] - buffer << "#{last_message[:name] || "User"} said #{last_message[:content]} " + buffer << "#{last_message[:name] || "User"}: #{last_message[:content]} " message = { type: :user, content: buffer } upload_ids.concat(last_message[:upload_ids]) if last_message[:upload_ids].present? diff --git a/plugin.rb b/plugin.rb index 4b7ab20b..e3218d93 100644 --- a/plugin.rb +++ b/plugin.rb @@ -77,6 +77,8 @@ after_initialize do # do not autoload this cause we may have no namespace require_relative "discourse_automation/llm_triage" require_relative "discourse_automation/llm_report" + require_relative "discourse_automation/llm_tool_triage" + require_relative "discourse_automation/llm_persona_triage" add_admin_route("discourse_ai.title", "discourse-ai", { use_new_show_route: true }) diff --git a/spec/lib/completions/prompt_messages_builder_spec.rb b/spec/lib/completions/prompt_messages_builder_spec.rb index 7e758e7a..b162e39c 100644 --- a/spec/lib/completions/prompt_messages_builder_spec.rb +++ b/spec/lib/completions/prompt_messages_builder_spec.rb @@ -40,4 +40,28 @@ describe DiscourseAi::Completions::PromptMessagesBuilder do expected = [{ type: :user, content: "Alice: Echo 123 please\nJames: OK" }] expect(builder.to_a).to eq(expected) end + + it "should format messages for topic style" do + # Create a topic with tags + topic = Fabricate(:topic, title: "This is an Example Topic") + + # Add tags to the topic + topic.tags = [Fabricate(:tag, name: "tag1"), Fabricate(:tag, name: "tag2")] + topic.save! + + builder.topic = topic + builder.push(type: :user, content: "I like frogs", name: "Bob") + builder.push(type: :user, content: "How do I solve this?", name: "Alice") + + result = builder.to_a(style: :topic) + + content = result[0][:content] + + expect(content).to include("This is an Example Topic") + expect(content).to include("tag1") + expect(content).to include("tag2") + expect(content).to include("Bob: I like frogs") + expect(content).to include("Alice") + expect(content).to include("How do I solve this") + end end diff --git a/spec/lib/discourse_automation/llm_persona_triage_spec.rb b/spec/lib/discourse_automation/llm_persona_triage_spec.rb new file mode 100644 index 00000000..37e40193 --- /dev/null +++ b/spec/lib/discourse_automation/llm_persona_triage_spec.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true + +return if !defined?(DiscourseAutomation) + +describe DiscourseAi::Automation::LlmPersonaTriage do + fab!(:user) + fab!(:bot_user) { Fabricate(:user) } + + fab!(:llm_model) do + Fabricate(:llm_model, provider: "anthropic", name: "claude-3-opus", enabled_chat_bot: true) + end + + fab!(:ai_persona) do + persona = + Fabricate( + :ai_persona, + name: "Triage Helper", + description: "A persona that helps with triaging posts", + system_prompt: "You are a helpful assistant that triages posts", + default_llm: llm_model, + ) + + # Create the user for this persona + persona.update!(user_id: bot_user.id) + persona + end + + let(:automation) { Fabricate(:automation, script: "llm_persona_triage", enabled: true) } + + def add_automation_field(name, value, type: "text") + automation.fields.create!( + component: type, + name: name, + metadata: { + value: value, + }, + target: "script", + ) + end + + before do + SiteSetting.ai_bot_enabled = true + SiteSetting.ai_bot_allowed_groups = "#{Group::AUTO_GROUPS[:trust_level_0]}" + + add_automation_field("persona", ai_persona.id, type: "choices") + add_automation_field("whisper", false, type: "boolean") + end + + it "can respond to a post using the specified persona" do + post = Fabricate(:post, raw: "This is a test post that needs triage") + + response_text = "I've analyzed your post and can help with that." + + DiscourseAi::Completions::Llm.with_prepared_responses([response_text]) do + automation.running_in_background! + automation.trigger!({ "post" => post }) + end + + topic = post.topic.reload + last_post = topic.posts.order(:post_number).last + + expect(topic.posts.count).to eq(2) + + # Verify that the response was posted by the persona's user + expect(last_post.user_id).to eq(bot_user.id) + expect(last_post.raw).to eq(response_text) + expect(last_post.post_type).to eq(Post.types[:regular]) # Not a whisper + end + + it "can respond with a whisper when configured to do so" do + add_automation_field("whisper", true, type: "boolean") + post = Fabricate(:post, raw: "This is another test post for triage") + + response_text = "Staff-only response to your post." + + DiscourseAi::Completions::Llm.with_prepared_responses([response_text]) do + automation.running_in_background! + automation.trigger!({ "post" => post }) + end + + topic = post.topic.reload + last_post = topic.posts.order(:post_number).last + + # Verify that the response is a whisper + expect(last_post.user_id).to eq(bot_user.id) + expect(last_post.raw).to eq(response_text) + expect(last_post.post_type).to eq(Post.types[:whisper]) # This should be a whisper + end + + it "does not respond to posts made by bots" do + bot = Fabricate(:bot) + bot_post = Fabricate(:post, user: bot, raw: "This is a bot post") + + # The automation should not trigger for bot posts + DiscourseAi::Completions::Llm.with_prepared_responses(["Response"]) do + automation.running_in_background! + automation.trigger!({ "post" => bot_post }) + end + + # Verify no new post was created + expect(bot_post.topic.reload.posts.count).to eq(1) + end + + it "handles errors gracefully" do + post = Fabricate(:post, raw: "Error-triggering post") + + # Set up to cause an error + ai_persona.update!(user_id: nil) + + # Should not raise an error + expect { + automation.running_in_background! + automation.trigger!({ "post" => post }) + }.not_to raise_error + + # Verify no new post was created + expect(post.topic.reload.posts.count).to eq(1) + end + + it "passes topic metadata in context when responding to topic" do + # Create a category and tags for the test + category = Fabricate(:category, name: "Test Category") + tag1 = Fabricate(:tag, name: "test-tag") + tag2 = Fabricate(:tag, name: "support") + + # Create a topic with category and tags + topic = + Fabricate( + :topic, + title: "Important Question About Feature", + category: category, + tags: [tag1, tag2], + user: user, + ) + + # Create a post in that topic + _post = + Fabricate( + :post, + topic: topic, + user: user, + raw: "This is a test post in a categorized and tagged topic", + ) + + post2 = + Fabricate(:post, topic: topic, user: user, raw: "This is another post in the same topic") + + # Capture the prompt sent to the LLM to verify it contains metadata + prompt = nil + + DiscourseAi::Completions::Llm.with_prepared_responses( + ["I've analyzed your question"], + ) do |_, _, _prompts| + automation.running_in_background! + automation.trigger!({ "post" => post2 }) + prompt = _prompts.first + end + + context = prompt.messages[1][:content] # The second message should be the triage prompt + + # Verify that topic metadata is included in the context + expect(context).to include("Important Question About Feature") + expect(context).to include("Test Category") + expect(context).to include("test-tag") + expect(context).to include("support") + end + + it "passes private message metadata in context when responding to PM" do + # Create a private message topic + pm_topic = Fabricate(:private_message_topic, user: user, title: "Important PM") + + # Create initial PM post + pm_post = + Fabricate( + :post, + topic: pm_topic, + user: user, + raw: "This is a private message that needs triage", + ) + + # Create a follow-up post + pm_post2 = + Fabricate( + :post, + topic: pm_topic, + user: user, + raw: "Adding more context to my private message", + ) + + # Capture the prompt sent to the LLM + prompt = nil + + DiscourseAi::Completions::Llm.with_prepared_responses( + ["I've received your private message"], + ) do |_, _, _prompts| + automation.running_in_background! + automation.trigger!({ "post" => pm_post2 }) + prompt = _prompts.first + end + + context = prompt.messages[1][:content] + + # Verify that PM metadata is included in the context + expect(context).to include("Important PM") + expect(context).to include(pm_post.raw) + expect(context).to include(pm_post2.raw) + end +end diff --git a/spec/lib/discourse_automation/llm_tool_triage_spec.rb b/spec/lib/discourse_automation/llm_tool_triage_spec.rb new file mode 100644 index 00000000..8baa8dd0 --- /dev/null +++ b/spec/lib/discourse_automation/llm_tool_triage_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe DiscourseAi::Automation::LlmToolTriage do + fab!(:solver) { Fabricate(:user) } + fab!(:new_user) { Fabricate(:user, trust_level: TrustLevel[0], created_at: 1.day.ago) } + fab!(:topic) { Fabricate(:topic, user: new_user) } + fab!(:post) { Fabricate(:post, topic: topic, user: new_user, raw: "How do I reset my password?") } + fab!(:llm_model) + fab!(:ai_persona) do + persona = Fabricate(:ai_persona, default_llm: llm_model) + persona.create_user + persona + end + + fab!(:tool) do + tool_script = <<~JS + function invoke(params) { + const postId = context.post_id; + const post = discourse.getPost(postId); + const user = discourse.getUser(post.user_id); + + if (user.trust_level > 0) { + return { + processed: false, + reason: "User is not new" + }; + } + + const helper = discourse.getPersona("#{ai_persona.name}"); + const answer = helper.respondTo({ post_id: post.id }); + + return { + answer: answer, + processed: true, + reason: "answered question" + }; + } + JS + + AiTool.create!( + name: "New User Question Answerer", + tool_name: "new_user_question_answerer", + description: "Automatically answers questions from new users when possible", + parameters: [], # No parameters as required by llm_tool_triage + script: tool_script, + created_by_id: Discourse.system_user.id, + summary: "Answers new user questions", + enabled: true, + ) + end + + before do + SiteSetting.discourse_ai_enabled = true + SiteSetting.ai_bot_enabled = true + end + + it "It is able to answer new user questions" do + result = nil + DiscourseAi::Completions::Llm.with_prepared_responses( + ["this is how you reset your password"], + ) { result = described_class.handle(post: post, tool_id: tool.id) } + expect(result["processed"]).to eq(true) + response = post.topic.reload.posts.order(:post_number).last + expect(response.raw).to eq("this is how you reset your password") + end + + it "Is able to respond as a whisper if instructed" do + # Create a tool with a script that explicitly requests a whisper response + whisper_tool = + AiTool.create!( + name: "Whisper Triage Tool", + tool_name: "whisper_triage_tool", + description: "Responds with whispers to moderation issues", + parameters: [], + script: <<~JS, + function invoke(params) { + const postId = context.post_id; + const post = discourse.getPost(postId); + + const helper = discourse.getPersona("#{ai_persona.name}"); + // Pass instructions to make response a whisper + const answer = helper.respondTo({ + post_id: post.id, + instructions: "Respond as a whisper for moderators only", + whisper: true + }); + + return { + answer: answer, + processed: true, + reason: "responded with whisper" + }; + } + JS + created_by_id: Discourse.system_user.id, + summary: "Responds with whispers", + enabled: true, + ) + + result = nil + DiscourseAi::Completions::Llm.with_prepared_responses( + ["This moderation note is only visible to staff"], + ) { result = described_class.handle(post: post, tool_id: whisper_tool.id) } + + expect(result["processed"]).to eq(true) + response = post.topic.reload.posts.order(:post_number).last + expect(response.raw).to eq("This moderation note is only visible to staff") + # Check that the response is indeed a whisper + expect(response.post_type).to eq(Post.types[:whisper]) + end +end diff --git a/spec/lib/modules/ai_bot/playground_spec.rb b/spec/lib/modules/ai_bot/playground_spec.rb index 3bcd4c12..6261aef8 100644 --- a/spec/lib/modules/ai_bot/playground_spec.rb +++ b/spec/lib/modules/ai_bot/playground_spec.rb @@ -141,18 +141,20 @@ RSpec.describe DiscourseAi::AiBot::Playground do it "can force usage of a tool" do tool_name = "custom-#{custom_tool.id}" ai_persona.update!(tools: [[tool_name, nil, true]], forced_tool_count: 1) - responses = [tool_call, "custom tool did stuff (maybe)"] + responses = [tool_call, ["custom tool did stuff (maybe)"], ["new PM title"]] prompts = nil reply_post = nil + private_message = Fabricate(:private_message_topic, user: user) + DiscourseAi::Completions::Llm.with_prepared_responses(responses) do |_, _, _prompts| - new_post = Fabricate(:post, raw: "Can you use the custom tool?") + new_post = Fabricate(:post, raw: "Can you use the custom tool?", topic: private_message) reply_post = playground.reply_to(new_post) prompts = _prompts end - expect(prompts.length).to eq(2) + expect(prompts.length).to eq(3) expect(prompts[0].tool_choice).to eq("search") expect(prompts[1].tool_choice).to eq(nil) @@ -381,7 +383,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do }}} Your instructions: - #{user.username} said Hello + #{user.username}: Hello TEXT expect(content.strip).to eq(expected) diff --git a/spec/models/ai_tool_spec.rb b/spec/models/ai_tool_spec.rb index 1d7135cb..66a826a9 100644 --- a/spec/models/ai_tool_spec.rb +++ b/spec/models/ai_tool_spec.rb @@ -188,6 +188,40 @@ RSpec.describe AiTool do expect(result).to eq("Hello") end + it "is able to run llm completions" do + script = <<~JS + function invoke(params) { + return llm.generate("question two") + llm.generate( + { messages: [ + { type: "system", content: "system message" }, + { type: "user", content: "user message" } + ]} + ); + } + JS + + tool = create_tool(script: script) + + result = nil + prompts = nil + responses = ["Hello ", "World"] + + DiscourseAi::Completions::Llm.with_prepared_responses(responses) do |_, _, _prompts| + runner = tool.runner({}, llm: llm, bot_user: nil, context: {}) + result = runner.invoke + prompts = _prompts + end + + prompt = + DiscourseAi::Completions::Prompt.new( + "system message", + messages: [{ type: :user, content: "user message" }], + ) + expect(result).to eq("Hello World") + expect(prompts[0]).to eq("question two") + expect(prompts[1]).to eq(prompt) + end + it "can timeout slow JS" do script = <<~JS function invoke(params) {