sources/ldap: add forward deletion option

Author: gergosimonyi

authentik/core/migrations/0049_groupsourceconnection_validated_by_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.9 on 2025-05-27 10:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("authentik_core", "0048_delete_oldauthenticatedsession_content_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="groupsourceconnection",
+            name="validated_by",
+            field=models.UUIDField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="usersourceconnection",
+            name="validated_by",
+            field=models.UUIDField(
+                blank=True, help_text="Helper field for batch deletions", null=True
+            ),
+        ),
+    ]

authentik/core/models.py

@@ -841,6 +841,9 @@ class UserSourceConnection(SerializerModel, CreatedUpdatedModel):
     user = models.ForeignKey(User, on_delete=models.CASCADE)
     source = models.ForeignKey(Source, on_delete=models.CASCADE)
     identifier = models.TextField()
+    validated_by = models.UUIDField(
+        null=True, blank=True, help_text=_("Helper field for batch deletions")
+    )
 
     objects = InheritanceManager()
 
@@ -866,6 +869,7 @@ class GroupSourceConnection(SerializerModel, CreatedUpdatedModel):
     group = models.ForeignKey(Group, on_delete=models.CASCADE)
     source = models.ForeignKey(Source, on_delete=models.CASCADE)
     identifier = models.TextField()
+    validated_by = models.UUIDField(null=True, blank=True)
 
     objects = InheritanceManager()
 

authentik/sources/ldap/api.py

@@ -111,6 +111,7 @@ class Meta:
             "sync_parent_group",
             "connectivity",
             "lookup_groups_from_user",
+            "delete_not_found_objects",
         ]
         extra_kwargs = {"bind_password": {"write_only": True}}
 
@@ -147,6 +148,7 @@ class LDAPSourceViewSet(UsedByMixin, ModelViewSet):
         "user_property_mappings",
         "group_property_mappings",
         "lookup_groups_from_user",
+        "delete_not_found_objects",
     ]
     search_fields = ["name", "slug"]
     ordering = ["name"]

authentik/sources/ldap/migrations/0009_ldapsource_delete_not_found_objects.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.9 on 2025-05-26 15:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("authentik_sources_ldap", "0008_groupldapsourceconnection_userldapsourceconnection"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ldapsource",
+            name="delete_not_found_objects",
+            field=models.BooleanField(
+                blank=True,
+                default=False,
+                help_text="Deletes authentik users and groups which were previously supplied by this source, but are now missing from it.",
+            ),
+        ),
+    ]

authentik/sources/ldap/models.py

@@ -137,6 +137,15 @@ class LDAPSource(Source):
         ),
     )
 
+    delete_not_found_objects = models.BooleanField(
+        default=False,
+        blank=True,
+        help_text=_(
+            "Deletes authentik users and groups which were previously supplied by this source, "
+            "but are now missing from it."
+        ),
+    )
+
     @property
     def component(self) -> str:
         return "ak-source-ldap-form"

authentik/sources/ldap/sync/base.py

@@ -9,7 +9,7 @@
 from authentik.core.sources.mapper import SourceMapper
 from authentik.lib.config import CONFIG
 from authentik.lib.sync.mapper import PropertyMappingManager
-from authentik.sources.ldap.models import LDAPSource
+from authentik.sources.ldap.models import LDAPSource, flatten
 
 
 class BaseLDAPSynchronizer:
@@ -77,6 +77,12 @@ def get_objects(self, **kwargs) -> Generator:
         """Get objects from LDAP, implemented in subclass"""
         raise NotImplementedError()
 
+    def get_identifier(self, object):
+        attributes = object.get("attributes", {})
+        if not attributes.get(self._source.object_uniqueness_field):
+            return
+        return flatten(attributes[self._source.object_uniqueness_field])
+
     def search_paginator(  # noqa: PLR0913
         self,
         search_base,

authentik/sources/ldap/sync/forward_delete_groups.py

@@ -0,0 +1,58 @@
+from collections.abc import Generator
+from itertools import batched
+from uuid import uuid4
+
+from ldap3 import SUBTREE
+
+from authentik.core.models import Group, GroupSourceConnection
+from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+from authentik.sources.ldap.sync.forward_delete_users import DELETE_CHUNK_SIZE, UPDATE_CHUNK_SIZE
+
+
+class GroupLDAPForwardDeletion(BaseLDAPSynchronizer):
+    """Delete LDAP Groups from authentik"""
+
+    @staticmethod
+    def name() -> str:
+        return "group_deletions"
+
+    def get_objects(self, **kwargs) -> Generator:
+        if not self._source.sync_groups:
+            self.message("Group syncing is disabled for this Source")
+            return iter(())
+
+        uuid = uuid4()
+        groups = self._source.connection().extend.standard.paged_search(
+            search_base=self.base_dn_groups,
+            search_filter=self._source.group_object_filter,
+            search_scope=SUBTREE,
+            attributes=[self._source.object_uniqueness_field],
+            generator=True,
+            **kwargs,
+        )
+        for batch in batched(groups, UPDATE_CHUNK_SIZE, strict=False):
+            identifiers = []
+            for group in batch:
+                if identifier := self.get_identifier(group):
+                    identifiers.append(identifier)
+            GroupSourceConnection.objects.filter(identifier__in=identifiers).update(
+                validated_by=uuid
+            )
+
+        return batched(
+            Group.objects.filter(groupsourceconnection__source=self._source)
+            .exclude(groupsourceconnection__validated_by=uuid)
+            .values_list("pk", flat=True)
+            .iterator(chunk_size=DELETE_CHUNK_SIZE),
+            DELETE_CHUNK_SIZE,
+            strict=False,
+        )
+
+    def sync(self, group_pks: tuple) -> int:
+        """Delete authentik groups"""
+        if not self._source.sync_groups:
+            self.message("Group syncing is disabled for this Source")
+            return -1
+        self._logger.debug("Deleting groups", group_pks=group_pks)
+        _, deleted_per_type = Group.objects.filter(pk__in=group_pks).delete()
+        return deleted_per_type.get(Group._meta.label, 0)

authentik/sources/ldap/sync/forward_delete_users.py

@@ -0,0 +1,60 @@
+from collections.abc import Generator
+from itertools import batched
+from uuid import uuid4
+
+from ldap3 import SUBTREE
+
+from authentik.core.models import User, UserSourceConnection
+from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+
+UPDATE_CHUNK_SIZE = 10_000
+DELETE_CHUNK_SIZE = 50
+
+
+class UserLDAPForwardDeletion(BaseLDAPSynchronizer):
+    """Delete LDAP Users from authentik"""
+
+    @staticmethod
+    def name() -> str:
+        return "user_deletions"
+
+    def get_objects(self, **kwargs) -> Generator:
+        if not self._source.sync_users:
+            self.message("User syncing is disabled for this Source")
+            return iter(())
+
+        uuid = uuid4()
+        users = self._source.connection().extend.standard.paged_search(
+            search_base=self.base_dn_users,
+            search_filter=self._source.user_object_filter,
+            search_scope=SUBTREE,
+            attributes=[self._source.object_uniqueness_field],
+            generator=True,
+            **kwargs,
+        )
+        for batch in batched(users, UPDATE_CHUNK_SIZE, strict=False):
+            identifiers = []
+            for user in batch:
+                if identifier := self.get_identifier(user):
+                    identifiers.append(identifier)
+            UserSourceConnection.objects.filter(identifier__in=identifiers).update(
+                validated_by=uuid
+            )
+
+        return batched(
+            User.objects.filter(usersourceconnection__source=self._source)
+            .exclude(usersourceconnection__validated_by=uuid)
+            .values_list("pk", flat=True)
+            .iterator(chunk_size=DELETE_CHUNK_SIZE),
+            DELETE_CHUNK_SIZE,
+            strict=False,
+        )
+
+    def sync(self, user_pks: tuple) -> int:
+        """Delete authentik users"""
+        if not self._source.sync_users:
+            self.message("User syncing is disabled for this Source")
+            return -1
+        self._logger.debug("Deleting users", user_pks=user_pks)
+        _, deleted_per_type = User.objects.filter(pk__in=user_pks).delete()
+        return deleted_per_type.get(User._meta.label, 0)

authentik/sources/ldap/tasks.py

@@ -17,6 +17,8 @@
 from authentik.root.celery import CELERY_APP
 from authentik.sources.ldap.models import LDAPSource
 from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+from authentik.sources.ldap.sync.forward_delete_groups import GroupLDAPForwardDeletion
+from authentik.sources.ldap.sync.forward_delete_users import UserLDAPForwardDeletion
 from authentik.sources.ldap.sync.groups import GroupLDAPSynchronizer
 from authentik.sources.ldap.sync.membership import MembershipLDAPSynchronizer
 from authentik.sources.ldap.sync.users import UserLDAPSynchronizer
@@ -52,11 +54,11 @@ def ldap_connectivity_check(pk: str | None = None):
 
 
 @CELERY_APP.task(
-    # We take the configured hours timeout time by 2.5 as we run user and
-    # group in parallel and then membership, so 2x is to cover the serial tasks,
+    # We take the configured hours timeout time by 3.5 as we run user and
+    # group in parallel and then membership, then deletions, so 3x is to cover the serial tasks,
     # and 0.5x on top of that to give some more leeway
-    soft_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 2.5,
-    task_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 2.5,
+    soft_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 3.5,
+    task_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 3.5,
 )
 def ldap_sync_single(source_pk: str):
     """Sync a single source"""
@@ -79,6 +81,25 @@ def ldap_sync_single(source_pk: str):
             group(
                 ldap_sync_paginator(source, MembershipLDAPSynchronizer),
             ),
+            # Finally, deletions. What we'd really like to do here is something like
+            # ```
+            # user_identifiers = <ldap query>
+            # User.objects.exclude(
+            #     usersourceconnection__identifier__in=user_uniqueness_identifiers,
+            # ).delete()
+            # ```
+            # This runs into performance issues in large installations. So instead we spread the
+            # work out into three steps:
+            # 1. Get every object from the LDAP source.
+            # 2. Mark every object as "safe" in the database. This is quick, but any error could
+            #    mean deleting users which should not be deleted, so we do it immediately, in
+            #    large chunks, and only queue the deletion step afterwards.
+            # 3. Delete every unmarked item. This is slow, so we spread it over many tasks in
+            #    small chunks.
+            group(
+                ldap_sync_paginator(source, UserLDAPForwardDeletion)
+                + ldap_sync_paginator(source, GroupLDAPForwardDeletion),
+            ),
         )
         task()
 

blueprints/schema.json

@@ -8180,6 +8180,11 @@
                     "type": "boolean",
                     "title": "Lookup groups from user",
                     "description": "Lookup group membership based on a user attribute instead of a group attribute. This allows nested group resolution on systems like FreeIPA and Active Directory"
+                },
+                "delete_not_found_objects": {
+                    "type": "boolean",
+                    "title": "Delete not found objects",
+                    "description": "Deletes authentik users and groups which were previously supplied by this source, but are now missing from it."
                 }
             },
             "required": []