From cf45e6884c93612e630d022f7273efc2d735a444 Mon Sep 17 00:00:00 2001
From: Sam <sam.saffron@gmail.com>
Date: Thu, 8 May 2025 12:51:36 +1000
Subject: [PATCH] FIX: persona triage should be logged to automation (#1326)

We were logging persona triage as "bot" in logs, causing some
confusions around real world usage

This amends it so we log usage to "automation - AUTOMATION NAME"
---
 lib/ai_bot/playground.rb                      |  6 ++-
 lib/automation/llm_persona_triage.rb          |  1 +
 .../llm_persona_triage_spec.rb                | 46 +++++++++++++++----
 3 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/lib/ai_bot/playground.rb b/lib/ai_bot/playground.rb
index 93190086..63aa8967 100644
--- a/lib/ai_bot/playground.rb
+++ b/lib/ai_bot/playground.rb
@@ -190,7 +190,8 @@ module DiscourseAi
         add_user_to_pm: false,
         stream_reply: false,
         auto_set_title: false,
-        silent_mode: false
+        silent_mode: false,
+        feature_name: nil
       )
         ai_persona = AiPersona.find_by(id: persona_id)
         raise Discourse::InvalidParameters.new(:persona_id) if !ai_persona
@@ -210,6 +211,7 @@ module DiscourseAi
           stream_reply: stream_reply,
           auto_set_title: auto_set_title,
           silent_mode: silent_mode,
+          feature_name: feature_name,
         )
       rescue => e
         if Rails.env.test?
@@ -380,6 +382,7 @@ module DiscourseAi
         stream_reply: nil,
         auto_set_title: true,
         silent_mode: false,
+        feature_name: nil,
         &blk
       )
         # this is a multithreading issue
@@ -414,6 +417,7 @@ module DiscourseAi
           DiscourseAi::Personas::BotContext.new(
             post: post,
             custom_instructions: custom_instructions,
+            feature_name: feature_name,
             messages:
               DiscourseAi::Completions::PromptMessagesBuilder.messages_from_post(
                 post,
diff --git a/lib/automation/llm_persona_triage.rb b/lib/automation/llm_persona_triage.rb
index abf7fa6d..dbbdbc6a 100644
--- a/lib/automation/llm_persona_triage.rb
+++ b/lib/automation/llm_persona_triage.rb
@@ -8,6 +8,7 @@ module DiscourseAi
           persona_id: persona_id,
           whisper: whisper,
           silent_mode: silent_mode,
+          feature_name: "automation - #{automation&.name}",
         )
       rescue => e
         Discourse.warn_exception(
diff --git a/spec/lib/discourse_automation/llm_persona_triage_spec.rb b/spec/lib/discourse_automation/llm_persona_triage_spec.rb
index da8b92df..8ea2123e 100644
--- a/spec/lib/discourse_automation/llm_persona_triage_spec.rb
+++ b/spec/lib/discourse_automation/llm_persona_triage_spec.rb
@@ -6,9 +6,7 @@ describe DiscourseAi::Automation::LlmPersonaTriage do
   fab!(:user)
   fab!(:bot_user) { Fabricate(:user) }
 
-  fab!(:llm_model) do
-    Fabricate(:llm_model, provider: "anthropic", name: "claude-3-opus", enabled_chat_bot: true)
-  end
+  fab!(:llm_model) { Fabricate(:anthropic_model, name: "claude-3-opus", enabled_chat_bot: true) }
 
   fab!(:ai_persona) do
     persona =
@@ -25,7 +23,9 @@ describe DiscourseAi::Automation::LlmPersonaTriage do
     persona
   end
 
-  let(:automation) { Fabricate(:automation, script: "llm_persona_triage", enabled: true) }
+  let(:automation) do
+    Fabricate(:automation, name: "my automation", script: "llm_persona_triage", enabled: true)
+  end
 
   def add_automation_field(name, value, type: "text")
     automation.fields.create!(
@@ -49,12 +49,40 @@ describe DiscourseAi::Automation::LlmPersonaTriage do
   it "can respond to a post using the specified persona" do
     post = Fabricate(:post, raw: "This is a test post that needs triage")
 
-    response_text = "I've analyzed your post and can help with that."
+    response_text = "I analyzed your post and can help with that."
 
-    DiscourseAi::Completions::Llm.with_prepared_responses([response_text]) do
-      automation.running_in_background!
-      automation.trigger!({ "post" => post })
-    end
+    body = (<<~STRING).strip
+      event: message_start
+      data: {"type": "message_start", "message": {"id": "msg_1nZdL29xx5MUA1yADyHTEsnR8uuvGzszyY", "type": "message", "role": "assistant", "content": [], "model": "claude-3-opus-20240229", "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 25, "output_tokens": 1}}}
+
+      event: content_block_start
+      data: {"type": "content_block_start", "index":0, "content_block": {"type": "text", "text": ""}}
+
+      event: ping
+      data: {"type": "ping"}
+
+      event: content_block_delta
+      data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "#{response_text}"}}
+
+      event: content_block_stop
+      data: {"type": "content_block_stop", "index": 0}
+
+      event: message_delta
+      data: {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence":null, "usage":{"output_tokens": 15}}}
+
+      event: message_stop
+      data: {"type": "message_stop"}
+    STRING
+
+    stub_request(:post, "https://api.anthropic.com/v1/messages").to_return(body: body)
+
+    automation.running_in_background!
+    automation.trigger!({ "post" => post })
+
+    log = AiApiAuditLog.last
+    expect(log).to be_present
+    expect(log.user_id).to eq(post.user_id)
+    expect(log.feature_name).to eq("automation - #{automation.name}")
 
     topic = post.topic.reload
     last_post = topic.posts.order(:post_number).last