PERF: Use DB for topic enumeration tasks (#7)

Use SQL statements versus loading topics into memory to allow handling of large datasets.

assets/javascripts/discourse/controllers/knowledge-explorer.js.es6

@@ -58,10 +58,7 @@ export default Ember.Controller.extend({
 
   @discourseComputed("loadMoreUrl")
   canLoadMore(loadMoreUrl) {
-    if (loadMoreUrl === null || this.isLoadingMore) {
+    return loadMoreUrl === null ? false : true;
-      return false;
-    }
-    return true;
   },
 
   @discourseComputed("searchTerm")
@@ -69,9 +66,9 @@ export default Ember.Controller.extend({
     return !!searchTerm;
   },
 
-  @discourseComputed("isSearching", "topics")
+  @discourseComputed("isSearching", "model")
-  searchCount(isSearching, topics) {
+  searchCount(isSearching, model) {
-    if (isSearching) return topics.length;
+    if (isSearching) return model.search_count;
   },
 
   emptySearchResults: Ember.computed.equal("searchCount", 0),
@@ -182,7 +179,7 @@ export default Ember.Controller.extend({
     },
 
     loadMore() {
-      if (this.canLoadMore) {
+      if (this.canLoadMore && !this.isLoadingMore) {
         this.set("isLoadingMore", true);
 
         KnowledgeExplorer.loadMore(this.loadMoreUrl).then(result => {

assets/javascripts/discourse/models/knowledge-explorer.js.es6

@@ -42,7 +42,19 @@ export default {
   },
 
   loadMore(loadMoreUrl) {
-    return ajax(loadMoreUrl);
+    let promise = ajax(loadMoreUrl);
+
+    promise = promise.then(data => {
+      data.topics.topic_list.topics = data.topics.topic_list.topics.map(
+        topic => {
+          topic = Topic.create(topic);
+          return topic;
+        }
+      );
+      return data;
+    });
+
+    return promise;
   },
 
   getTopic

lib/knowledge_explorer/query.rb

@@ -72,6 +72,7 @@ module KnowledgeExplorer
               pd.search_data @@ #{escaped_ts_query}
           )
         SQL
+        search_count = results.count
       end
 
       if @filters[:order] == "title"
@@ -88,8 +89,15 @@ module KnowledgeExplorer
         end
       end
 
-      tags = tag_count(results)
+      # conduct a second set of joins so we don't mess up the count
-      categories = categories_count(results)
+      count_query = results.joins <<~SQL
+        INNER JOIN topic_tags tt2 ON tt2.topic_id = topics.id
+        INNER JOIN tags t2 ON t2.id = tt2.tag_id
+      SQL
+      tags = count_query.group('t2.name').count
+      tags = create_tags_object(tags)
+      categories = results.where('topics.category_id IS NOT NULL').group('topics.category_id').count
+      categories = create_categories_object(categories)
 
       results_length = results.length
 
@@ -103,7 +111,7 @@ module KnowledgeExplorer
         end_of_list = true if results_length < @limit
       end
 
-      results = results[offset...page_range]
+      results = results.offset(offset).limit(@limit) #results[offset...page_range]
 
       # assemble the object
       topic_query = tq.create_list(:knowledge_explorer, { unordered: true }, results)
@@ -116,45 +124,33 @@ module KnowledgeExplorer
         topic_list['load_more_url'] = nil
       end
 
-      { tags: tags, categories: categories, topics: topic_list }
+      { tags: tags, categories: categories, topics: topic_list, search_count: search_count }
     end
 
-    def tag_count(results)
+    def create_tags_object(tags)
-      tags = []
+      tags_object = []
 
-      results.each do |topic|
+      tags.each do |tag|
-        topic.tags.each do |tag|
+        active = @filters[:tags].include?(tag[0]) if @filters[:tags]
-          active = @filters[:tags].include?(tag.name) if @filters[:tags]
+        tags_object << { id: tag[0], count: tag[1], active: active || false }
-          if tags.none? { |item| item[:id].to_s == tag.name }
-            tags << { id: tag.name, count: 1, active: active || false }
-          else
-            tag_index = tags.index(tags.find { |item| item[:id].to_s == tag.name })
-            tags[tag_index][:count] += 1
-          end
-        end
       end
 
       allowed_tags = DiscourseTagging.filter_allowed_tags(Guardian.new(@user)).map(&:name)
 
-      tags = tags.select { |tag| allowed_tags.include?(tag[:id]) }
+      tags_object = tags_object.select { |tag| allowed_tags.include?(tag[:id]) }
 
-      tags.sort_by { |tag| [tag[:active] ? 0 : 1, -tag[:count]] }
+      tags_object.sort_by { |tag| [tag[:active] ? 0 : 1, -tag[:count]] }
     end
 
-    def categories_count(results)
+    def create_categories_object(categories)
-      categories = []
+      categories_object = []
 
-      results.each do |topic|
+      categories.each do |category|
-        active = @filters[:category].include?(topic.category_id.to_s) if @filters[:category]
+        active = @filters[:category].include?(category[0].to_s) if @filters[:category]
-        if categories.none? { |item| item[:id] == topic.category_id }
+        categories_object << { id: category[0], count: category[1], active: active || false }
-          categories << { id: topic.category_id, count: 1, active: active || false }
-        else
-          category_index = categories.index(categories.find { |item| item[:id] == topic.category_id })
-          categories[category_index][:count] += 1
-        end
       end
 
-      categories.sort_by { |category| [category[:active] ? 0 : 1, -category[:count]] }
+      categories_object.sort_by { |category| [category[:active] ? 0 : 1, -category[:count]] }
     end
 
     def load_more_url