REFACTOR: Handle queries in more robust, customizable way

* Create custom query lib file

* Get topic list by category

* Get topic list with both categories and tags

* Count tags and pass back to controller in object

* Filter topic list by param-passed tag list

* FIX: Correctly serialize topic list data

* Filter results by search term (title only

* Debug commit

* Working multi-tag filtering

* FIX: case insensitive search terms

* Begin refactor of front end for new api changes

* REFACTOR: Use model for refreshing data
Instead of just using a route, which introduces full page refreshes, use
the route to pull the data initially, then update it using a model as to
refresh only the relevant parts of the page.

* Working topic load

* FIX: Visual alignment

* Refactor tests to follow new patterns

* Fixes suggested by eviltrout

* FEATURE: Load more topics

* FIX: Paginate records on return to the front end in a better fashion

* FIX: Prevent loadMore while loading more

* Fix pagination of topics to truncate list properly

* Inherit rubocop from discourse

* Make rubocop happynated

* Set list to unordered

.gitignore

@@ -0,0 +1 @@
+ .rubocop-https---raw-githubusercontent-com-discourse-discourse-master--rubocop-yml

.rubocop.yml

@@ -0,0 +1 @@
+ inherit_from: https://raw.githubusercontent.com/discourse/discourse/master/.rubocop.yml

app/controllers/knowledge_explorer/knowledge_explorer_controller.rb

@@ -3,135 +3,18 @@
 module KnowledgeExplorer
   class KnowledgeExplorerController < ApplicationController
     requires_plugin 'knowledge-explorer'
-    before_action :init_guardian
 
     def index
-
       filters = {
         tags: params[:tags],
-        category: params[:category]
+        category: params[:category],
+        search_term: params[:search],
+        page: params[:page]
       }
 
-      if filters[:category]
+      query = KnowledgeExplorer::Query.new(current_user, filters).list
-       category_topic_lists = get_topics_from_categories(category_by_filter(filters[:category]))
-      else
-       category_topic_lists = get_topics_from_categories(knowledge_explorer_categories)
-      end
 
-      tag_topic_lists = get_topics_from_tags(knowledge_explorer_tags)
+      render json: query
-
-      # Deduplicate results
-
-      topics = []
-
-      category_topic_lists.each do |list|
-        list[:topic_list][:topics].each do |t| 
-          if topics.none?{|item| item[:id] == t[:id]}
-            if t[:id] != Category.find(t[:category_id]).topic_id
-              topics << t
-            end
-          end
-        end
-      end
-      tag_topic_lists.each do |list|
-        list[:topic_list][:topics].each do |t| 
-          if topics.none?{|item| item[:id] == t[:id]}
-            topics << t
-          end
-        end
-      end
-
-      if filters[:tags]
-        tag_filter = filters[:tags].split(' ')
-        topics = topics.select { |topic| (topic[:tags] & tag_filter).size >= 1}
-      end
-
-      topics = count_tags(topics)
-
-      render json: topics
-    end
-
-    def get_topics_from_categories(categories)
-      category_topic_lists = []
-
-      categories.each do |c|
-        if topic_list = TopicQuery.new(current_user, category: c.id, no_subcategories: true).list_latest
-          category_topic_lists << TopicListSerializer.new(topic_list, scope: @guardian).as_json
-        end
-      end
-
-      category_topic_lists
-    end
-
-    def get_topics_from_tags(tags)
-      tag_topic_lists = []
-
-      tags.each do |t|
-        if topic_list = TopicQuery.new(current_user, tags: t.name).list_latest
-          tag_topic_lists << TopicListSerializer.new(topic_list, scope: @guardian).as_json
-        end
-      end
-
-      tag_topic_lists
-    end
-
-    def count_tags(topics)
-      tags = []
-
-      topics.each do |topic|
-        topic[:tags].each do |tag| 
-          if params[:tags]
-            active = params[:tags].include?(tag)
-          end
-          if tags.none? { |item| item[:id].to_s == tag }
-            tags << { id: tag, count: 1 , active: active || false }
-          else
-            tag_index = tags.index(tags.find { |item| item[:id].to_s == tag })
-            tags[tag_index][:count] += 1
-          end
-        end
-      end
-
-      { tags: tags, topics: topics }
-    end
-
-    private
-
-    def init_guardian
-      @guardian = Guardian.new(current_user)
-    end
-    
-    def knowledge_explorer_categories
-      selected_categories = SiteSetting.knowledge_explorer_categories.split("|")
-
-      if selected_categories
-        categories = Category.where('id IN (?)', selected_categories)
-
-        return categories.select { |c| @guardian.can_see_category?(c) }
-      end
-    end
-
-    def knowledge_explorer_tags
-      selected_tags = SiteSetting.knowledge_explorer_tags.split("|")
-
-      if selected_tags
-        return Tag.where('name IN (?)', selected_tags)
-      end
-    end
-
-    def category_by_filter(category_filter)
-      selected_category = category_filter
-
-      category = Category.where('slug IN (?)', selected_category)
-
-      category.select { |c| @guardian.can_see_category?(c) }
-    end
-
-    def tags_by_filter(tags)
-      selected_tags = tags.split(' ')
-      if (selected_tags)
-        return Tag.where('name IN (?)', selected_tags)
-      end
     end
   end
 end

assets/javascripts/discourse/controllers/knowledge-explorer.js.es6

@@ -2,64 +2,117 @@ import {
   default as computed,
   observes
 } from "ember-addons/ember-computed-decorators";
-import knowledgeExplorer from "discourse/plugins/discourse-knowledge-explorer/discourse/models/knowledge-explorer";
+import KnowledgeExplorer from "discourse/plugins/discourse-knowledge-explorer/discourse/models/knowledge-explorer";
 
 export default Ember.Controller.extend({
   application: Ember.inject.controller(),
   queryParams: {
     filterCategory: "category",
     filterTags: "tags",
+    searchTerm: "search",
     selectedTopic: "topic"
   },
+  isLoading: false,
+  isLoadingMore: false,
+  loadMoreUrl: Ember.computed.alias("model.topics.load_more_url"),
+  @computed("loadMoreUrl")
+  canLoadMore(loadMoreUrl) {
+    if (loadMoreUrl === null || this.isLoadingMore) {
+      return false;
+    }
+    return true;
+  },
+  isTopicLoading: false,
+  topics: Ember.computed.alias("model.topics.topic_list.topics"),
+  tags: Ember.computed.readOnly("model.tags"),
   filterTags: null,
   filterCategory: null,
-
   searchTerm: null,
-  searchResults: null,
 
   selectedTopic: null,
+  topic: null,
 
-  searchCount: Ember.computed.readOnly("searchResults.length"),
+  @computed("searchTerm")
-  emptySearchResults: Ember.computed.equal("searchCount", 0),
+  isSearching(searchTerm) {
-
+    return !!searchTerm;
-  @computed("searchResults")
-  hasSearchResults(results) {
-    return !!results;
   },
 
+  @computed("isSearching", "topics")
+  searchCount(isSearching, topics) {
+    if (isSearching) return topics.length;
+  },
+  emptySearchResults: Ember.computed.equal("searchCount", 0),
+
   @computed("filterTags")
   filtered(filterTags) {
     return !!filterTags;
   },
 
   actions: {
-    setSelectedTopic(topicID) {
+    setSelectedTopic(topicId) {
-      this.set("selectedTopic", topicID);
+      this.set("isTopicLoading", true);
+      this.set("selectedTopic", topicId);
+      KnowledgeExplorer.getTopic(topicId).then(result => {
+        this.set("topic", result);
+        this.set("isTopicLoading", false);
+      });
     },
     updateSelectedTags(tag) {
       let filter = this.filterTags;
       if (filter && filter.includes(tag.id)) {
         filter = filter.replace(tag.id, "");
-        filter = filter.replace("++", "+");
+        filter = filter.replace("|", "|");
-        filter = filter.replace(/^\++|\++$/g, "");
+        filter = filter.replace(/^\|+|\|+$/g, "");
       } else if (filter) {
-        filter = `${filter}+${tag.id}`;
+        filter = `${filter}|${tag.id}`;
       } else {
         filter = tag.id;
       }
 
       this.set("filterTags", filter);
+      this.set("selectedTopic", null);
+      this.send("refreshModel");
     },
     performSearch(term) {
-      if (term.length < this.siteSettings.min_search_term_length) {
+      if (term === "") {
-        this.set("searchResults", null);
+        this.set("searchTerm", null);
+        this.send("refreshModel");
         return false;
       }
 
-      const tags = this.get("filterTags") || null;
+      if (term.length < this.siteSettings.min_search_term_length) {
+        return false;
+      }
 
-      knowledgeExplorer.search(term, tags).then(result => {
+      this.set("searchTerm", term);
-        this.set("searchResults", result.topics || []);
+      this.set("selectedTopic", null);
+      this.send("refreshModel");
+    },
+    loadMore() {
+      if (this.canLoadMore) {
+        this.set("isLoadingMore", true);
+
+        KnowledgeExplorer.loadMore(this.loadMoreUrl).then(result => {
+          let topics = this.topics;
+
+          topics = topics.concat(result.topics.topic_list.topics);
+
+          this.set("topics", topics);
+          this.set("loadMoreUrl", result.topics.load_more_url || null);
+          this.set("isLoadingMore", false);
+        });
+      }
+    },
+    refreshModel() {
+      this.set("isLoading", true);
+      const params = this.getProperties(
+        "filterCategory",
+        "filterTags",
+        "searchTerm"
+      );
+      KnowledgeExplorer.list(params).then(result => {
+        this.set("model", result);
+        this.set("isLoading", false);
       });
     }
   }

assets/javascripts/discourse/models/knowledge-explorer.js.es6

@@ -1,12 +1,18 @@
 import { ajax } from "discourse/lib/ajax";
 
 export default {
-  //write as one liner
+  list(params) {
-  //use + instead of space for query param
+    let filters = [];
-  search(filter, tags) {
+    if (params.filterTags) filters.push(`tags=${params.filterTags}`);
-    let params = [filter];
+    if (params.searchTerm) filters.push(`search=${params.searchTerm}`);
-    if (tags) params.push(`tags:${tags}`);
+    if (params.page) filters.push(`page=${params.page}`);
-    const endpoint = `/search.json?q=in:kb ${params.join(" ")}`;
+
-    return ajax(endpoint);
+    return ajax(`/knowledge-explorer.json?${filters.join("&")}`);
+  },
+  loadMore(loadMoreUrl) {
+    return ajax(loadMoreUrl);
+  },
+  getTopic(id) {
+    return ajax(`/t/${id}.json`);
   }
 };

assets/javascripts/discourse/routes/knowledge-explorer.js.es6

@@ -1,37 +1,8 @@
 import { ajax } from "discourse/lib/ajax";
+import KnowledgeExplorer from "discourse/plugins/discourse-knowledge-explorer/discourse/models/knowledge-explorer";
 
 export default Ember.Route.extend({
-  queryParams: {
-    filterTags: {
-      refreshModel: true
-    },
-    selectedTopic: {
-      refreshModel: true
-    }
-  },
   model(params) {
-    if (params.filterTags) {
+    return KnowledgeExplorer.list(params);
-      const tags = params.filterTags;
-      return ajax(`/knowledge-explorer.json?tags=${tags}`);
-    } else if (params.selectedTopic) {
-      return ajax(`/t/${params.selectedTopic}.json`);
-    } else {
-      return ajax("/knowledge-explorer.json");
-    }
-  },
-
-  setupController(controller, model) {
-    if (model.tags && model.topics) {
-      controller.setProperties({
-        tags: model.tags,
-        topics: model.topics
-      });
-    } else {
-      controller.setProperties({
-        tags: [],
-        topics: [],
-        topic: model
-      });
-    }
   }
 });

assets/javascripts/discourse/templates/components/knowledge-explorer-topic-list.hbs

@@ -1,3 +1,4 @@
+{{#load-more selector=".topic-list tr" action=loadMore}}
 <table class="topic-list">
   <thead>
     <th>Topic</th>
@@ -23,4 +24,5 @@
     {{/each}}
   </tbody>
 </table>
-
+{{/load-more}}
+{{conditional-loading-spinner condition=loading}}

assets/javascripts/discourse/templates/components/knowledge-explorer-topic.hbs

@@ -1,5 +1,8 @@
-<a class="knowledge-explorer-nav-link return" href="/knowledge-explorer">Return to Knowledge Explorer</a>
+{{#link-to 'knowledgeExplorer' (query-params topic=null) class='knowledge-explorer-nav-link return'}}
+  {{i18n 'knowledge_explorer.topic.back'}}
+{{/link-to}}
 <div class="topic-content">
+  <h1>{{topic.title}}</h1>
   {{{originalPostContent}}}
 </div>
-<a class="knowledge-explorer-nav-link more" href="/t/{{topic.id}}">View the discussion on this topic</a>
+<a class="knowledge-explorer-nav-link more" href="/t/{{topic.id}}">{{i18n 'knowledge_explorer.topic.navigate_to_topic'}}</a>

assets/javascripts/discourse/templates/knowledge-explorer.hbs

@@ -1,12 +1,10 @@
 <div class="knowledge-explorer">
-  {{#if selectedTopic}}
-    {{knowledge-explorer-topic topic=topic}}
-  {{else}}
   <div class="knowledge-explorer-search">
     {{knowledge-explorer-search 
       onSearch=(action "performSearch")
     }}
   </div>
+  {{#conditional-loading-spinner condition=isLoading}}
     <div class="knowledge-explorer-browse">
       <div class="knowledge-explorer-tags">
         {{#each tags as |tag|}}
@@ -16,24 +14,29 @@
             }}
         {{/each}}
       </div>
+      {{#if selectedTopic}}
+        {{#conditional-loading-spinner condition=isTopicLoading}}
+          {{knowledge-explorer-topic topic=topic}}
+        {{/conditional-loading-spinner}}
+      {{else}}
         <div class="knowledge-explorer-results">
-      {{#if hasSearchResults}}
+        {{#if isSearching}}
           {{#if emptySearchResults}}
             <div class="result-count no-result">{{i18n 'search.no_results'}}</div>
           {{else}}
             <div class="result-count">{{i18n 'knowledge_explorer.search.results' count=searchCount}}</div>
-          {{knowledge-explorer-topic-list 
-            topics=searchResults 
-            selectTopic=(action "setSelectedTopic")
-          }}
           {{/if}}
-      {{else}}
+        {{/if}}
+        {{#unless emptySearchResults}}
           {{knowledge-explorer-topic-list 
             topics=topics 
             selectTopic=(action "setSelectedTopic")
+            loadMore=(action "loadMore")
+            loading=isLoadingMore
           }}
-      {{/if}}
+        {{/unless}}
-      </div>
         </div>
       {{/if}}
+    </div>
+  {{/conditional-loading-spinner}}
 </div>

assets/stylesheets/common/knowledge-explorer.scss

@@ -1,26 +1,4 @@
 .knowledge-explorer {
-  .knowledge-explorer-topic {
-    width: 80%;
-    margin: 0 auto;
-    .knowledge-explorer-nav-link {
-      font-weight: 700;
-      &.return {
-        font-size: $font-down-1;
-        &::before {
-          content: "«";
-          margin-right: 5px;
-        }
-      }
-      &.more {
-        float: right;
-        font-size: $font-up-1;
-        padding: 10px 0;
-        &::after {
-          content: "»";
-          margin-left: 5px;
-        }
-      }
-    }
   }
  .knowledge-explorer-search {
   align-items: center;
@@ -36,6 +14,11 @@
  }
  .knowledge-explorer-browse {
    display: flex;
+   .loading-container {
+     display: flex;
+     flex-basis: 100%;
+     padding: 10px 0 10px 10px;
+   }
    .knowledge-explorer-results {
      display: flex;
      flex-direction: column;
@@ -96,5 +79,30 @@
         padding-right: 5px;
       }
     }
+  .knowledge-explorer-topic {
+    display: flex;
+    flex-direction: column;
+    .knowledge-explorer-nav-link {
+      font-weight: 700;
+      &.return {
+        font-size: $font-down-1;
+        &::before {
+          content: "«";
+          margin-right: 5px;
+        }
+      }
+      &.more {
+        margin-left: auto;
+        font-size: $font-up-1;
+        padding: 10px 0;
+        &::after {
+          content: "»";
+          margin-left: 5px;
+        }
+      }
+    }
+    .topic-content {
+      padding-top: 10px;
+    }
  } 
 }

config/locales/client.en.yml

@@ -6,3 +6,6 @@ en:
         results: 
           one: "%{count} result found"
           other: "%{count} results found"
+      topic:
+        back: "Go back"
+        navigate_to_topic: "View the discussion on this topic"

config/routes.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_dependency "knowledge_explorer_constraint"
+require_dependency 'knowledge_explorer_constraint'
 
 KnowledgeExplorer::Engine.routes.draw do
-  get "/" => "knowledge_explorer#index", constraints: KnowledgeExplorerConstraint.new
+  get '/' => 'knowledge_explorer#index', constraints: KnowledgeExplorerConstraint.new
-  get ".json" => "knowledge_explorer#index", constraints: KnowledgeExplorerConstraint.new
+  get '.json' => 'knowledge_explorer#index', constraints: KnowledgeExplorerConstraint.new
 end

lib/knowledge_explorer/engine.rb

@@ -6,7 +6,7 @@ module ::KnowledgeExplorer
 
     config.after_initialize do
       Discourse::Application.routes.append do
-        mount ::KnowledgeExplorer::Engine, at: "/knowledge-explorer"
+        mount ::KnowledgeExplorer::Engine, at: '/knowledge-explorer'
       end
     end
   end

lib/knowledge_explorer/query.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module KnowledgeExplorer
+  class Query
+    def initialize(user = nil, filters = {})
+      @user = user
+      @filters = filters
+      @limit = 30
+    end
+
+    def self.categories
+      SiteSetting.knowledge_explorer_categories.split('|')
+    end
+
+    def self.tags
+      SiteSetting.knowledge_explorer_tags.split('|')
+    end
+
+    def list
+      # query for topics matching selected categories & tags
+      tq = TopicQuery.new(@user)
+      results = tq.latest_results(no_definitions: true, limit: false)
+      results = results.left_outer_joins(:tags)
+      results = results.where('category_id IN (?)', Query.categories).or(results.where('tags.name IN (?)', Query.tags))
+
+      # filter results by selected category
+      if @filters[:category].present?
+        results = results.where('category_id IN (?)', @filters[:category])
+      end
+
+      # filter results by selected tags
+      if @filters[:tags].present?
+        tag_filters = @filters[:tags].split('|')
+        tags_count = tag_filters.length
+        tag_filters = Tag.where_name(tag_filters).pluck(:id) unless Integer === tag_filters[0]
+
+        if tags_count == tag_filters.length
+          tag_filters.each_with_index do |tag, index|
+            sql_alias = ['t', index].join
+            results = results.joins("INNER JOIN topic_tags #{sql_alias} ON #{sql_alias}.topic_id = topics.id AND #{sql_alias}.tag_id = #{tag}")
+          end
+        else
+          results = results.none # don't return any results unless all tags exist in the database
+        end
+      end
+
+      # filter results by search term
+      if @filters[:search_term].present?
+        results = results.where('lower(title) LIKE ?', "%#{@filters[:search_term].downcase}%")
+      end
+
+      tags = tag_count(results)
+
+      results_length = results.length
+
+      if @filters[:page]
+        offset = @filters[:page].to_i * @limit
+        page_range = offset + @limit
+        end_of_list = true if page_range > results_length
+      else
+        offset = 0
+        page_range = @limit
+      end
+
+      results = results[offset...page_range]
+
+      # assemble the object
+      topic_query = tq.create_list(:knowledge_explorer, { unordered: true }, results)
+
+      topic_list = TopicListSerializer.new(topic_query, scope: Guardian.new(@user)).as_json
+
+      if end_of_list.nil?
+        topic_list['load_more_url'] = load_more_url
+      else
+        topic_list['load_more_url'] = nil
+      end
+
+      { tags: tags, topics: topic_list }
+    end
+
+    def tag_count(results)
+      tags = []
+
+      results.each do |topic|
+        topic.tags.each do |tag|
+          active = @filters[:tags].include?(tag.name) if @filters[:tags]
+          if tags.none? { |item| item[:id].to_s == tag.name }
+            tags << { id: tag.name, count: 1, active: active || false }
+          else
+            tag_index = tags.index(tags.find { |item| item[:id].to_s == tag.name })
+            tags[tag_index][:count] += 1
+          end
+        end
+      end
+
+      tags.sort_by { |tag| [tag[:active] ? 0 : 1, -tag[:count]] }
+    end
+
+    def load_more_url
+      filters = []
+
+      filters.push("tags=#{@filters[:tags]}") if @filters[:tags].present?
+      filters.push("category=#{@filters[:category]}") if @filters[:category].present?
+      filters.push("search=#{@filters[:search_term]}") if @filters[:search_term].present?
+
+      if @filters[:page].present?
+        filters.push("page=#{@filters[:page].to_i + 1}")
+      else
+        filters.push('page=1')
+      end
+
+      "/knowledge-explorer.json?#{filters.join('&')}"
+    end
+  end
+end

lib/knowledge_explorer_constraint.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class KnowledgeExplorerConstraint
-  def matches?(request)
+  def matches?(_request)
     SiteSetting.knowledge_explorer_enabled
   end
 end

plugin.rb

@@ -10,7 +10,8 @@ enabled_site_setting :knowledge_explorer_enabled
 register_asset 'stylesheets/common/knowledge-explorer.scss'
 register_asset 'stylesheets/mobile/knowledge-explorer.scss'
 
-load File.expand_path('../lib/knowledge_explorer/engine.rb', __FILE__)
+load File.expand_path('lib/knowledge_explorer/engine.rb', __dir__)
+load File.expand_path('lib/knowledge_explorer/query.rb', __dir__)
 
 after_initialize do
   require_dependency 'search'
@@ -18,12 +19,12 @@ after_initialize do
   if SiteSetting.knowledge_explorer_enabled
     if Search.respond_to? :advanced_filter
       Search.advanced_filter(/in:kb/) do |posts|
-        selected_categories = SiteSetting.knowledge_explorer_categories.split("|")
+        selected_categories = SiteSetting.knowledge_explorer_categories.split('|')
         if selected_categories
           categories = Category.where('id IN (?)', selected_categories).pluck(:id)
         end
 
-        selected_tags = SiteSetting.knowledge_explorer_tags.split("|")
+        selected_tags = SiteSetting.knowledge_explorer_tags.split('|')
         if selected_tags
           tags = Tag.where('name IN (?)', selected_tags).pluck(:id)
         end

spec/requests/knowledge_explorer_controller_spec.rb

@@ -23,13 +23,15 @@ describe KnowledgeExplorer::KnowledgeExplorerController do
         expect(response.status).to eq(200)
 
         json = JSON.parse(response.body)
+        tags = json['tags']
+        topics = json['topics']['topic_list']['topics']
 
-        expect(json['tags'].size).to eq(1)
+        expect(tags.size).to eq(1)
-        expect(json['topics'].size).to eq(2)
+        expect(topics.size).to eq(2)
       end
     end
 
-    context "when some knowledge explorer topics are private" do
+    context 'when some knowledge explorer topics are private' do
       let!(:group) { Fabricate(:group) }
       let!(:private_category) { Fabricate(:private_category, group: group) }
       let!(:private_topic) { Fabricate(:topic, category: private_category) }
@@ -42,8 +44,9 @@ describe KnowledgeExplorer::KnowledgeExplorerController do
         get '/knowledge-explorer.json'
 
         json = JSON.parse(response.body)
+        topics = json['topics']['topic_list']['topics']
 
-        expect(json['topics'].size).to eq(2)
+        expect(topics.size).to eq(2)
       end
 
       it 'should show topics when users have permissions' do
@@ -53,8 +56,9 @@ describe KnowledgeExplorer::KnowledgeExplorerController do
         get '/knowledge-explorer.json'
 
         json = JSON.parse(response.body)
+        topics = json['topics']['topic_list']['topics']
 
-        expect(json['topics'].size).to eq(3)
+        expect(topics.size).to eq(3)
       end
     end
 
@@ -65,9 +69,11 @@ describe KnowledgeExplorer::KnowledgeExplorerController do
         expect(response.status).to eq(200)
 
         json = JSON.parse(response.body)
+        tags = json['tags']
+        topics = json['topics']['topic_list']['topics']
 
-        expect(json['tags'].size).to eq(1)
+        expect(tags.size).to eq(1)
-        expect(json['topics'].size).to eq(1)
+        expect(topics.size).to eq(1)
       end
     end
   end