From 45386920cadeddba004fa23c8e0b749185d94259 Mon Sep 17 00:00:00 2001
From: Bianca Nenciu <nbianca@users.noreply.github.com>
Date: Wed, 13 Jul 2022 19:53:56 +0300
Subject: [PATCH] FIX: Use new robots.txt API (#101)

The old robots.txt was invalid because "User-agent: *" was found twice
in robots.txt. Using the API will generate correct robots.txt.
---
 .../connectors/robots_txt_index/docs.html.erb    |  4 ----
 plugin.rb                                        | 13 +++++++++++++
 spec/requests/robots_txt_controller_spec.rb      | 16 ++++++++++++++++
 3 files changed, 29 insertions(+), 4 deletions(-)
 delete mode 100644 app/views/connectors/robots_txt_index/docs.html.erb
 create mode 100644 spec/requests/robots_txt_controller_spec.rb

diff --git a/app/views/connectors/robots_txt_index/docs.html.erb b/app/views/connectors/robots_txt_index/docs.html.erb
deleted file mode 100644
index bf55d3c..0000000
--- a/app/views/connectors/robots_txt_index/docs.html.erb
+++ /dev/null
@@ -1,4 +0,0 @@
-<%- if SiteSetting.docs_enabled%>
-User-agent: *
-Disallow: /docs/
-<% end %>
diff --git a/plugin.rb b/plugin.rb
index baa363b..563e588 100644
--- a/plugin.rb
+++ b/plugin.rb
@@ -44,4 +44,17 @@ after_initialize do
   add_to_class(:topic_query, :list_docs_topics) do
     default_results(@options)
   end
+
+  on(:robots_info) do |robots_info|
+    robots_info[:agents] ||= []
+
+    any_user_agent = robots_info[:agents].find { |info| info[:name] == "*" }
+    if !any_user_agent
+      any_user_agent = { name: "*" }
+      robots_info[:agents] << any_user_agent
+    end
+
+    any_user_agent[:disallow] ||= []
+    any_user_agent[:disallow] << "/docs/"
+  end
 end
diff --git a/spec/requests/robots_txt_controller_spec.rb b/spec/requests/robots_txt_controller_spec.rb
new file mode 100644
index 0000000..3a8536d
--- /dev/null
+++ b/spec/requests/robots_txt_controller_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe RobotsTxtController do
+  before do
+    SiteSetting.docs_enabled = true
+  end
+
+  it 'adds /docs/ to robots.txt' do
+    get '/robots.txt'
+
+    expect(response.body).to include('User-agent: *')
+    expect(response.body).to include('Disallow: /docs/')
+  end
+end