diff --git a/lib/assigner.rb b/lib/assigner.rb
index 48c1a99..e4d083c 100644
--- a/lib/assigner.rb
+++ b/lib/assigner.rb
@@ -266,7 +266,7 @@ class ::Assigner
           topic.invite_group(@assigned_by, assign_to)
         end
       else
-        unless topic.topic_allowed_users.exists?(user_id: assign_to.id)
+        unless topic.all_allowed_users.exists?(id: assign_to.id)
           guardian.ensure_can_invite_to!(topic)
           topic.invite(@assigned_by, assign_to.username)
         end
diff --git a/spec/lib/assigner_spec.rb b/spec/lib/assigner_spec.rb
index 06813f7..a26dc77 100644
--- a/spec/lib/assigner_spec.rb
+++ b/spec/lib/assigner_spec.rb
@@ -725,6 +725,14 @@ RSpec.describe Assigner do
       expect(topic.allowed_users).to include(user)
     end
 
+    it "doesn't invite user to the PM when already a member of an allowed group" do
+      user = Fabricate(:user)
+      assign_allowed_group.add(user)
+      topic.allowed_groups << assign_allowed_group
+      assigner.assign(user)
+      expect(topic.allowed_users).not_to include(user)
+    end
+
     it "invites group to the PM" do
       group =
         Fabricate(