discourse
/
discourse-ai
mirror of https://github.com/discourse/discourse-ai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Add summarization type to eval (#1138) 

Adds `type: summarization` for topic summarization eval: https://github.com/discourse/discourse-ai-evals/pull/4
This commit is contained in:
Keegan George 2025-02-20 09:07:23 -08:00 committed by GitHub
parent 70248ccfca
commit 1f9f330ce2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 22 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
evals/lib/eval.rb
View File

@ -60,6 +60,8 @@ class DiscourseAi::Evals::Eval
prompt_call(llm, **args) prompt_call(llm, **args)
when "edit_artifact" when "edit_artifact"
edit_artifact(llm, **args) edit_artifact(llm, **args)
when "summarization"
summarization(llm, **args)
end end
if expected_output if expected_output
@ -312,4 +314,24 @@ class DiscourseAi::Evals::Eval
rescue StandardError rescue StandardError
false false
end end
def summarization(llm, input:)
topic =
Topic.new(
category: Category.last,
title: "Eval topic for topic summarization",
id: -99,
user_id: Discourse.system_user.id,
)
Post.new(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
strategy =
DiscourseAi::Summarization::FoldContent.new(
llm.llm_proxy,
DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
)
summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
summary.summarized_text
end
end end