FIX: guide GPT 3.5 better (#77)

* FIX: guide GPT 3.5 better

This limits search results to 10 cause we were blowing the whole token
budget on search results, additionally it includes a quick exchange at
the start of a session to try and guide GPT 3.5 to follow instructions

Sadly GPT 3.5 drifts off very quickly but this does improve stuff a bit.

It also attempts to correct some issues with anthropic, though it still is
surprisingly hard to ground

* add status:public, this is a bit of a hack but ensures that we can search
for any filter provided

* fix specs

lib/modules/ai_bot/anthropic_bot.rb

@@ -15,6 +15,26 @@ module DiscourseAi
         7500 # https://console.anthropic.com/docs/prompt-design#what-is-a-prompt
       end
 
+      def get_delta(partial, context)
+        context[:pos] ||= 0
+
+        full = partial[:completion]
+        delta = full[context[:pos]..-1]
+
+        context[:pos] = full.length
+
+        if !context[:processed]
+          delta = ""
+          index = full.index("Assistant: ")
+          if index
+            delta = full[index + 11..-1]
+            context[:processed] = true
+          end
+        end
+
+        delta
+      end
+
       private
 
       def build_message(poster_username, content, system: false)
@@ -27,10 +47,6 @@ module DiscourseAi
         "claude-v1"
       end
 
-      def update_with_delta(_, partial)
-        partial[:completion]
-      end
-
       def get_updated_title(prompt)
         DiscourseAi::Inference::AnthropicCompletions.perform!(
           prompt,

lib/modules/ai_bot/bot.rb

@@ -3,6 +3,8 @@
 module DiscourseAi
   module AiBot
     class Bot
+      attr_reader :bot_user
+
       BOT_NOT_FOUND = Class.new(StandardError)
       MAX_COMPLETIONS = 3
 
@@ -50,13 +52,14 @@ module DiscourseAi
           end
 
         redis_stream_key = nil
-        reply = bot_reply_post ? bot_reply_post.raw : ""
+        reply = +(bot_reply_post ? bot_reply_post.raw.dup : "")
         start = Time.now
 
         setup_cancel = false
+        context = {}
 
         submit_prompt(prompt, prefer_low_cost: prefer_low_cost) do |partial, cancel|
-          reply = update_with_delta(reply, partial)
+          reply << get_delta(partial, context)
 
           if redis_stream_key && !Discourse.redis.get(redis_stream_key)
             cancel&.call
@@ -92,6 +95,7 @@ module DiscourseAi
 
         if bot_reply_post
           publish_update(bot_reply_post, done: true)
+
           bot_reply_post.revise(
             bot_user,
             { raw: reply },
@@ -154,6 +158,9 @@ module DiscourseAi
             memo.unshift(build_message(username, raw))
           end
 
+        # we need this to ground the model (especially GPT-3.5)
+        messages.unshift(build_message(bot_user.username, "!echo 1"))
+        messages.unshift(build_message("user", "please echo 1"))
         messages.unshift(build_message(bot_user.username, rendered_system_prompt, system: true))
         messages
       end
@@ -205,7 +212,9 @@ module DiscourseAi
           The participants in this conversation are: #{post.topic.allowed_users.map(&:username).join(", ")}
           The date now is: #{Time.zone.now}, much has changed since you were trained.
 
-          You can complete some tasks using multiple steps and have access to some special commands!
+          You can complete some tasks using !commands.
+
+          NEVER ask user to issue !commands, they have no access, only you do.
 
           #{available_commands.map(&:desc).join("\n")}
 
@@ -233,9 +242,11 @@ module DiscourseAi
         raise NotImplemented
       end
 
-      protected
+      def get_delta(partial, context)
+        raise NotImplemented
+      end
 
-      attr_reader :bot_user
+      protected
 
       def get_updated_title(prompt)
         raise NotImplemented
@@ -245,10 +256,6 @@ module DiscourseAi
         raise NotImplemented
       end
 
-      def get_delta_from(partial)
-        raise NotImplemented
-      end
-
       def conversation_context(post)
         context =
           post

lib/modules/ai_bot/commands/command.rb

@@ -29,6 +29,10 @@ module DiscourseAi
           @args = args
         end
 
+        def bot
+          @bot ||= DiscourseAi::AiBot::Bot.as(bot_user)
+        end
+
         def standalone?
           false
         end
@@ -81,6 +85,9 @@ module DiscourseAi
 
           raw << custom_raw if custom_raw.present?
 
+          replacement = "!#{self.class.name} #{args}"
+          raw = post.raw.sub(replacement, raw) if post.raw.include?(replacement)
+
           if chain_next_response
             post.raw = raw
             post.save!(validate: false)

lib/modules/ai_bot/commands/search_command.rb

@@ -93,10 +93,18 @@ module DiscourseAi::AiBot::Commands
 
       @last_query = search_string
       results =
-        Search.execute(search_string.to_s, search_type: :full_page, guardian: Guardian.new())
+        Search.execute(
+          search_string.to_s + " status:public",
+          search_type: :full_page,
+          guardian: Guardian.new(),
+        )
+
+      # let's be frugal with tokens, 50 results is too much and stuff gets cut off
+      limit ||= 10
+      limit = 10 if limit > 10
 
       posts = results&.posts || []
-      posts = posts[0..limit - 1] if limit
+      posts = posts[0..limit - 1]
 
       @last_num_results = posts.length
 

lib/modules/ai_bot/commands/summarize_command.rb

@@ -84,10 +84,6 @@ module DiscourseAi::AiBot::Commands
       false
     end
 
-    def bot
-      @bot ||= DiscourseAi::AiBot::Bot.as(bot_user)
-    end
-
     def summarize(data, guidance, topic)
       text = +""
       data.each do |id, post_number, raw, username|

lib/modules/ai_bot/open_ai_bot.rb

@@ -75,8 +75,8 @@ module DiscourseAi
         "gpt-3.5-turbo"
       end
 
-      def update_with_delta(current_delta, partial)
+      def get_delta(partial, _context)
-        current_delta + partial.dig(:choices, 0, :delta, :content).to_s
+        partial.dig(:choices, 0, :delta, :content).to_s
       end
 
       def get_updated_title(prompt)

spec/lib/modules/ai_bot/anthropic_bot_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::AiBot::AnthropicBot do
+  describe "#update_with_delta" do
+    def bot_user
+      User.find(DiscourseAi::AiBot::EntryPoint::GPT4_ID)
+    end
+
+    subject { described_class.new(bot_user) }
+
+    describe "get_delta" do
+      it "can properly remove Assistant prefix" do
+        context = {}
+        reply = +""
+
+        reply << subject.get_delta({ completion: "\n\nAssist" }, context)
+        expect(reply).to eq("")
+
+        reply << subject.get_delta({ completion: "\n\nAssistant: test" }, context)
+        expect(reply).to eq("test")
+
+        reply << subject.get_delta({ completion: "\n\nAssistant: test\nworld" }, context)
+        expect(reply).to eq("test\nworld")
+      end
+    end
+  end
+end

spec/lib/modules/ai_bot/bot_spec.rb

@@ -43,7 +43,7 @@ RSpec.describe DiscourseAi::AiBot::Bot do
     it "can respond to !search" do
       bot.system_prompt_style!(:simple)
 
-      expected_response = "!search test search"
+      expected_response = "ok, searching...\n!search test search"
 
       prompt = bot.bot_prompt_with_topic_context(second_post)
 
@@ -65,12 +65,14 @@ RSpec.describe DiscourseAi::AiBot::Bot do
       bot.reply_to(second_post)
 
       last = second_post.topic.posts.order("id desc").first
-      expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now")
 
       expect(last.raw).to include("<details>")
       expect(last.raw).to include("<summary>Search</summary>")
       expect(last.raw).not_to include("translation missing")
+      expect(last.raw).to include("ok, searching...")
       expect(last.raw).to include("We are done now")
+
+      expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now")
     end
   end
 

spec/lib/modules/ai_bot/commands/search_command_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
       post1 = Fabricate(:post)
       search = described_class.new(bot_user, post1)
 
-      results = search.process("order:fake")
+      results = search.process("order:fake ABDDCDCEDGDG")
       expect(results).to eq("No results found")
     end
 
@@ -29,6 +29,10 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
 
       # title + 2 rows
       expect(results.split("\n").length).to eq(3)
+
+      # just searching for everything
+      results = search.process("order:latest_topic")
+      expect(results.split("\n").length).to be > 1
     end
   end
 end

spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb

@@ -63,7 +63,8 @@ RSpec.describe Jobs::CreateAiReply do
     end
 
     context "when chatting with Claude from Anthropic" do
-      let(:deltas) { expected_response.split(" ").map { |w| "#{w} " } }
+      let(:claude_response) { "Assistant: #{expected_response}" }
+      let(:deltas) { claude_response.split(" ").map { |w| "#{w} " } }
 
       before do
         bot_user = User.find(DiscourseAi::AiBot::EntryPoint::CLAUDE_V1_ID)

spec/lib/modules/ai_bot/open_ai_bot_spec.rb

@@ -20,7 +20,7 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
       it "includes it in the prompt" do
         prompt_messages = subject.bot_prompt_with_topic_context(post_1)
 
-        post_1_message = prompt_messages[1]
+        post_1_message = prompt_messages[-1]
 
         expect(post_1_message[:role]).to eq("user")
         expect(post_1_message[:content]).to eq("#{post_1.user.username}: #{post_body(1)}")
@@ -33,11 +33,11 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
       it "trims the prompt" do
         prompt_messages = subject.bot_prompt_with_topic_context(post_1)
 
-        expect(prompt_messages[0][:role]).to eq("system")
+        expect(prompt_messages[-2][:role]).to eq("assistant")
-        expect(prompt_messages[1][:role]).to eq("user")
+        expect(prompt_messages[-1][:role]).to eq("user")
         # trimming is tricky... it needs to account for system message as
         # well... just make sure we trim for now
-        expect(prompt_messages[1][:content].length).to be < post_1.raw.length
+        expect(prompt_messages[-1][:content].length).to be < post_1.raw.length
       end
     end
 
@@ -51,14 +51,15 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
       it "includes them in the prompt respecting the post number order" do
         prompt_messages = subject.bot_prompt_with_topic_context(post_3)
 
-        expect(prompt_messages[1][:role]).to eq("user")
+        # negative cause we may have grounding prompts
-        expect(prompt_messages[1][:content]).to eq("#{post_1.username}: #{post_body(1)}")
+        expect(prompt_messages[-3][:role]).to eq("user")
+        expect(prompt_messages[-3][:content]).to eq("#{post_1.username}: #{post_body(1)}")
 
-        expect(prompt_messages[2][:role]).to eq("assistant")
+        expect(prompt_messages[-2][:role]).to eq("assistant")
-        expect(prompt_messages[2][:content]).to eq(post_body(2))
+        expect(prompt_messages[-2][:content]).to eq(post_body(2))
 
-        expect(prompt_messages[3][:role]).to eq("user")
+        expect(prompt_messages[-1][:role]).to eq("user")
-        expect(prompt_messages[3][:content]).to eq("#{post_3.username}: #{post_body(3)}")
+        expect(prompt_messages[-1][:content]).to eq("#{post_3.username}: #{post_body(3)}")
       end
     end
   end