DEV: Migration to remove old embeddings tables~ (#1067)

* DEV: Migration to remove old embeddings tables~

* Check for table existence

db/migrate/20250114160417_backfill_topic_embeddings.rb

@@ -3,6 +3,7 @@ class BackfillTopicEmbeddings < ActiveRecord::Migration[7.2]
   disable_ddl_transaction!
 
   def up
+    if table_exists?(:ai_topic_embeddings)
       loop do
         count = execute(<<~SQL).cmd_tuples
         INSERT INTO ai_topics_embeddings (topic_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
@@ -23,6 +24,7 @@ class BackfillTopicEmbeddings < ActiveRecord::Migration[7.2]
         break if count == 0
       end
     end
+  end
 
   def down
     raise ActiveRecord::IrreversibleMigration

db/migrate/20250114160446_backfill_post_embeddings.rb

@@ -3,6 +3,7 @@ class BackfillPostEmbeddings < ActiveRecord::Migration[7.2]
   disable_ddl_transaction!
 
   def up
+    if table_exists?(:ai_post_embeddings)
       # Copy data from old tables to new tables in batches.
 
       loop do
@@ -25,6 +26,7 @@ class BackfillPostEmbeddings < ActiveRecord::Migration[7.2]
         break if count == 0
       end
     end
+  end
 
   def down
     raise ActiveRecord::IrreversibleMigration

db/migrate/20250114160500_backfill_rag_embeddings.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 class BackfillRagEmbeddings < ActiveRecord::Migration[7.2]
   def up
+    if table_exists?(:ai_document_fragment_embeddings)
       not_backfilled =
         DB.query_single("SELECT COUNT(*) FROM ai_document_fragments_embeddings").first.to_i == 0
 
@@ -12,6 +13,7 @@ class BackfillRagEmbeddings < ActiveRecord::Migration[7.2]
         SQL
       end
     end
+  end
 
   def down
     raise ActiveRecord::IrreversibleMigration

db/post_migrate/20250114184356_drop_old_embedding_tables2.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+class DropOldEmbeddingTables2 < ActiveRecord::Migration[7.2]
+  def up
+    # Copy rag embeddings created during deploy.
+    execute <<~SQL
+        INSERT INTO ai_document_fragments_embeddings (rag_document_fragment_id, model_id, model_version, strategy_id, strategy_version, digest, embeddings, created_at, updated_at)
+        (
+          SELECT  old_table.*
+          FROM ai_document_fragment_embeddings old_table
+          LEFT OUTER JOIN ai_document_fragments_embeddings target ON (
+            target.model_id = old_table.model_id AND
+            target.strategy_id = old_table.strategy_id AND
+            target.rag_document_fragment_id = old_table.rag_document_fragment_id
+          )
+          WHERE target.rag_document_fragment_id IS NULL
+        )
+      SQL
+
+    execute <<~SQL
+        DROP INDEX IF EXISTS ai_topic_embeddings_1_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_2_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_3_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_4_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_5_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_6_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_7_1_search_bit;
+        DROP INDEX IF EXISTS ai_topic_embeddings_8_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_1_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_2_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_3_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_4_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_5_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_6_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_7_1_search_bit;
+        DROP INDEX IF EXISTS ai_post_embeddings_8_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_1_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_2_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_3_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_4_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_5_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_6_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_7_1_search_bit;
+        DROP INDEX IF EXISTS ai_document_fragment_embeddings_8_1_search_bit;
+      SQL
+
+    drop_table :ai_topic_embeddings, if_exists: true
+    drop_table :ai_post_embeddings, if_exists: true
+    drop_table :ai_document_fragment_embeddings, if_exists: true
+  end
+
+  def down
+  end
+end