sources/ldap: add forward deletion option (#14718)

gergosimonyi · web-flow · commit c4bb19051d1e · 2025-05-28T13:22:59.000+02:00
* sources/ldap: add forward deletion option

* remove unnecessary `blank=True`

* clarify `validated_by` `help_text`

* add indices to `validated_by`

* factor out `get_identifier` everywhere and `get_attributes`

I don't know what that additional `in` check is for, but I'm not about
to find out.

* add tests for known good user and group

* fixup! add tests for known good user and group

* fixup! add tests for known good user and group
diff --git a/authentik/sources/ldap/api.py b/authentik/sources/ldap/api.py
@@ -111,6 +111,7 @@ class Meta:
             "sync_parent_group",
             "connectivity",
             "lookup_groups_from_user",
+            "delete_not_found_objects",
         ]
         extra_kwargs = {"bind_password": {"write_only": True}}
 
@@ -147,6 +148,7 @@ class LDAPSourceViewSet(UsedByMixin, ModelViewSet):
         "user_property_mappings",
         "group_property_mappings",
         "lookup_groups_from_user",
+        "delete_not_found_objects",
     ]
     search_fields = ["name", "slug"]
     ordering = ["name"]
diff --git a/authentik/sources/ldap/migrations/0009_groupldapsourceconnection_validated_by_and_more.py b/authentik/sources/ldap/migrations/0009_groupldapsourceconnection_validated_by_and_more.py
@@ -0,0 +1,48 @@
+# Generated by Django 5.1.9 on 2025-05-28 08:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("authentik_core", "0048_delete_oldauthenticatedsession_content_type"),
+        ("authentik_sources_ldap", "0008_groupldapsourceconnection_userldapsourceconnection"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="groupldapsourceconnection",
+            name="validated_by",
+            field=models.UUIDField(
+                blank=True,
+                help_text="Unique ID used while checking if this object still exists in the directory.",
+                null=True,
+            ),
+        ),
+        migrations.AddField(
+            model_name="ldapsource",
+            name="delete_not_found_objects",
+            field=models.BooleanField(
+                default=False,
+                help_text="Delete authentik users and groups which were previously supplied by this source, but are now missing from it.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="userldapsourceconnection",
+            name="validated_by",
+            field=models.UUIDField(
+                blank=True,
+                help_text="Unique ID used while checking if this object still exists in the directory.",
+                null=True,
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="groupldapsourceconnection",
+            index=models.Index(fields=["validated_by"], name="authentik_s_validat_b70447_idx"),
+        ),
+        migrations.AddIndex(
+            model_name="userldapsourceconnection",
+            index=models.Index(fields=["validated_by"], name="authentik_s_validat_ff2ebc_idx"),
+        ),
+    ]
diff --git a/authentik/sources/ldap/models.py b/authentik/sources/ldap/models.py
@@ -137,6 +137,14 @@ class LDAPSource(Source):
         ),
     )
 
+    delete_not_found_objects = models.BooleanField(
+        default=False,
+        help_text=_(
+            "Delete authentik users and groups which were previously supplied by this source, "
+            "but are now missing from it."
+        ),
+    )
+
     @property
     def component(self) -> str:
         return "ak-source-ldap-form"
@@ -321,6 +329,12 @@ class Meta:
 
 
 class UserLDAPSourceConnection(UserSourceConnection):
+    validated_by = models.UUIDField(
+        null=True,
+        blank=True,
+        help_text=_("Unique ID used while checking if this object still exists in the directory."),
+    )
+
     @property
     def serializer(self) -> type[Serializer]:
         from authentik.sources.ldap.api import (
@@ -332,9 +346,18 @@ def serializer(self) -> type[Serializer]:
     class Meta:
         verbose_name = _("User LDAP Source Connection")
         verbose_name_plural = _("User LDAP Source Connections")
+        indexes = [
+            models.Index(fields=["validated_by"]),
+        ]
 
 
 class GroupLDAPSourceConnection(GroupSourceConnection):
+    validated_by = models.UUIDField(
+        null=True,
+        blank=True,
+        help_text=_("Unique ID used while checking if this object still exists in the directory."),
+    )
+
     @property
     def serializer(self) -> type[Serializer]:
         from authentik.sources.ldap.api import (
@@ -346,3 +369,6 @@ def serializer(self) -> type[Serializer]:
     class Meta:
         verbose_name = _("Group LDAP Source Connection")
         verbose_name_plural = _("Group LDAP Source Connections")
+        indexes = [
+            models.Index(fields=["validated_by"]),
+        ]
diff --git a/authentik/sources/ldap/sync/base.py b/authentik/sources/ldap/sync/base.py
@@ -9,7 +9,7 @@
 from authentik.core.sources.mapper import SourceMapper
 from authentik.lib.config import CONFIG
 from authentik.lib.sync.mapper import PropertyMappingManager
-from authentik.sources.ldap.models import LDAPSource
+from authentik.sources.ldap.models import LDAPSource, flatten
 
 
 class BaseLDAPSynchronizer:
@@ -77,6 +77,16 @@ def get_objects(self, **kwargs) -> Generator:
         """Get objects from LDAP, implemented in subclass"""
         raise NotImplementedError()
 
+    def get_attributes(self, object):
+        if "attributes" not in object:
+            return
+        return object.get("attributes", {})
+
+    def get_identifier(self, attributes: dict):
+        if not attributes.get(self._source.object_uniqueness_field):
+            return
+        return flatten(attributes[self._source.object_uniqueness_field])
+
     def search_paginator(  # noqa: PLR0913
         self,
         search_base,
diff --git a/authentik/sources/ldap/sync/forward_delete_groups.py b/authentik/sources/ldap/sync/forward_delete_groups.py
@@ -0,0 +1,61 @@
+from collections.abc import Generator
+from itertools import batched
+from uuid import uuid4
+
+from ldap3 import SUBTREE
+
+from authentik.core.models import Group
+from authentik.sources.ldap.models import GroupLDAPSourceConnection
+from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+from authentik.sources.ldap.sync.forward_delete_users import DELETE_CHUNK_SIZE, UPDATE_CHUNK_SIZE
+
+
+class GroupLDAPForwardDeletion(BaseLDAPSynchronizer):
+    """Delete LDAP Groups from authentik"""
+
+    @staticmethod
+    def name() -> str:
+        return "group_deletions"
+
+    def get_objects(self, **kwargs) -> Generator:
+        if not self._source.sync_groups or not self._source.delete_not_found_objects:
+            self.message("Group syncing is disabled for this Source")
+            return iter(())
+
+        uuid = uuid4()
+        groups = self._source.connection().extend.standard.paged_search(
+            search_base=self.base_dn_groups,
+            search_filter=self._source.group_object_filter,
+            search_scope=SUBTREE,
+            attributes=[self._source.object_uniqueness_field],
+            generator=True,
+            **kwargs,
+        )
+        for batch in batched(groups, UPDATE_CHUNK_SIZE, strict=False):
+            identifiers = []
+            for group in batch:
+                if not (attributes := self.get_attributes(group)):
+                    continue
+                if identifier := self.get_identifier(attributes):
+                    identifiers.append(identifier)
+            GroupLDAPSourceConnection.objects.filter(identifier__in=identifiers).update(
+                validated_by=uuid
+            )
+
+        return batched(
+            GroupLDAPSourceConnection.objects.filter(source=self._source)
+            .exclude(validated_by=uuid)
+            .values_list("group", flat=True)
+            .iterator(chunk_size=DELETE_CHUNK_SIZE),
+            DELETE_CHUNK_SIZE,
+            strict=False,
+        )
+
+    def sync(self, group_pks: tuple) -> int:
+        """Delete authentik groups"""
+        if not self._source.sync_groups or not self._source.delete_not_found_objects:
+            self.message("Group syncing is disabled for this Source")
+            return -1
+        self._logger.debug("Deleting groups", group_pks=group_pks)
+        _, deleted_per_type = Group.objects.filter(pk__in=group_pks).delete()
+        return deleted_per_type.get(Group._meta.label, 0)
diff --git a/authentik/sources/ldap/sync/forward_delete_users.py b/authentik/sources/ldap/sync/forward_delete_users.py
@@ -0,0 +1,63 @@
+from collections.abc import Generator
+from itertools import batched
+from uuid import uuid4
+
+from ldap3 import SUBTREE
+
+from authentik.core.models import User
+from authentik.sources.ldap.models import UserLDAPSourceConnection
+from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+
+UPDATE_CHUNK_SIZE = 10_000
+DELETE_CHUNK_SIZE = 50
+
+
+class UserLDAPForwardDeletion(BaseLDAPSynchronizer):
+    """Delete LDAP Users from authentik"""
+
+    @staticmethod
+    def name() -> str:
+        return "user_deletions"
+
+    def get_objects(self, **kwargs) -> Generator:
+        if not self._source.sync_users or not self._source.delete_not_found_objects:
+            self.message("User syncing is disabled for this Source")
+            return iter(())
+
+        uuid = uuid4()
+        users = self._source.connection().extend.standard.paged_search(
+            search_base=self.base_dn_users,
+            search_filter=self._source.user_object_filter,
+            search_scope=SUBTREE,
+            attributes=[self._source.object_uniqueness_field],
+            generator=True,
+            **kwargs,
+        )
+        for batch in batched(users, UPDATE_CHUNK_SIZE, strict=False):
+            identifiers = []
+            for user in batch:
+                if not (attributes := self.get_attributes(user)):
+                    continue
+                if identifier := self.get_identifier(attributes):
+                    identifiers.append(identifier)
+            UserLDAPSourceConnection.objects.filter(identifier__in=identifiers).update(
+                validated_by=uuid
+            )
+
+        return batched(
+            UserLDAPSourceConnection.objects.filter(source=self._source)
+            .exclude(validated_by=uuid)
+            .values_list("user", flat=True)
+            .iterator(chunk_size=DELETE_CHUNK_SIZE),
+            DELETE_CHUNK_SIZE,
+            strict=False,
+        )
+
+    def sync(self, user_pks: tuple) -> int:
+        """Delete authentik users"""
+        if not self._source.sync_users or not self._source.delete_not_found_objects:
+            self.message("User syncing is disabled for this Source")
+            return -1
+        self._logger.debug("Deleting users", user_pks=user_pks)
+        _, deleted_per_type = User.objects.filter(pk__in=user_pks).delete()
+        return deleted_per_type.get(User._meta.label, 0)
diff --git a/authentik/sources/ldap/sync/groups.py b/authentik/sources/ldap/sync/groups.py
@@ -58,18 +58,16 @@ def sync(self, page_data: list) -> int:
             return -1
         group_count = 0
         for group in page_data:
-            if "attributes" not in group:
+            if (attributes := self.get_attributes(group)) is None:
                 continue
-            attributes = group.get("attributes", {})
             group_dn = flatten(flatten(group.get("entryDN", group.get("dn"))))
-            if not attributes.get(self._source.object_uniqueness_field):
+            if not (uniq := self.get_identifier(attributes)):
                 self.message(
                     f"Uniqueness field not found/not set in attributes: '{group_dn}'",
                     attributes=attributes.keys(),
                     dn=group_dn,
                 )
                 continue
-            uniq = flatten(attributes[self._source.object_uniqueness_field])
             try:
                 defaults = {
                     k: flatten(v)
diff --git a/authentik/sources/ldap/sync/membership.py b/authentik/sources/ldap/sync/membership.py
@@ -63,9 +63,9 @@ def sync(self, page_data: list) -> int:
                     group_member_dn = group_member.get("dn", {})
                     members.append(group_member_dn)
             else:
-                if "attributes" not in group:
+                if (attributes := self.get_attributes(group)) is None:
                     continue
-                members = group.get("attributes", {}).get(self._source.group_membership_field, [])
+                members = attributes.get(self._source.group_membership_field, [])
 
             ak_group = self.get_group(group)
             if not ak_group:
diff --git a/authentik/sources/ldap/sync/users.py b/authentik/sources/ldap/sync/users.py
@@ -60,18 +60,16 @@ def sync(self, page_data: list) -> int:
             return -1
         user_count = 0
         for user in page_data:
-            if "attributes" not in user:
+            if (attributes := self.get_attributes(user)) is None:
                 continue
-            attributes = user.get("attributes", {})
             user_dn = flatten(user.get("entryDN", user.get("dn")))
-            if not attributes.get(self._source.object_uniqueness_field):
+            if not (uniq := self.get_identifier(attributes)):
                 self.message(
                     f"Uniqueness field not found/not set in attributes: '{user_dn}'",
                     attributes=attributes.keys(),
                     dn=user_dn,
                 )
                 continue
-            uniq = flatten(attributes[self._source.object_uniqueness_field])
             try:
                 defaults = {
                     k: flatten(v)
diff --git a/authentik/sources/ldap/tasks.py b/authentik/sources/ldap/tasks.py
@@ -17,6 +17,8 @@
 from authentik.root.celery import CELERY_APP
 from authentik.sources.ldap.models import LDAPSource
 from authentik.sources.ldap.sync.base import BaseLDAPSynchronizer
+from authentik.sources.ldap.sync.forward_delete_groups import GroupLDAPForwardDeletion
+from authentik.sources.ldap.sync.forward_delete_users import UserLDAPForwardDeletion
 from authentik.sources.ldap.sync.groups import GroupLDAPSynchronizer
 from authentik.sources.ldap.sync.membership import MembershipLDAPSynchronizer
 from authentik.sources.ldap.sync.users import UserLDAPSynchronizer
@@ -52,11 +54,11 @@ def ldap_connectivity_check(pk: str | None = None):
 
 
 @CELERY_APP.task(
-    # We take the configured hours timeout time by 2.5 as we run user and
-    # group in parallel and then membership, so 2x is to cover the serial tasks,
+    # We take the configured hours timeout time by 3.5 as we run user and
+    # group in parallel and then membership, then deletions, so 3x is to cover the serial tasks,
     # and 0.5x on top of that to give some more leeway
-    soft_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 2.5,
-    task_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 2.5,
+    soft_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 3.5,
+    task_time_limit=(60 * 60 * CONFIG.get_int("ldap.task_timeout_hours")) * 3.5,
 )
 def ldap_sync_single(source_pk: str):
     """Sync a single source"""
@@ -79,6 +81,25 @@ def ldap_sync_single(source_pk: str):
             group(
                 ldap_sync_paginator(source, MembershipLDAPSynchronizer),
             ),
+            # Finally, deletions. What we'd really like to do here is something like
+            # ```
+            # user_identifiers = <ldap query>
+            # User.objects.exclude(
+            #     usersourceconnection__identifier__in=user_uniqueness_identifiers,
+            # ).delete()
+            # ```
+            # This runs into performance issues in large installations. So instead we spread the
+            # work out into three steps:
+            # 1. Get every object from the LDAP source.
+            # 2. Mark every object as "safe" in the database. This is quick, but any error could
+            #    mean deleting users which should not be deleted, so we do it immediately, in
+            #    large chunks, and only queue the deletion step afterwards.
+            # 3. Delete every unmarked item. This is slow, so we spread it over many tasks in
+            #    small chunks.
+            group(
+                ldap_sync_paginator(source, UserLDAPForwardDeletion)
+                + ldap_sync_paginator(source, GroupLDAPForwardDeletion),
+            ),
         )
         task()
 
diff --git a/authentik/sources/ldap/tests/mock_slapd.py b/authentik/sources/ldap/tests/mock_slapd.py
diff --git a/authentik/sources/ldap/tests/test_sync.py b/authentik/sources/ldap/tests/test_sync.py
diff --git a/blueprints/schema.json b/blueprints/schema.json
diff --git a/schema.yml b/schema.yml
diff --git a/web/src/admin/sources/ldap/LDAPSourceForm.ts b/web/src/admin/sources/ldap/LDAPSourceForm.ts

Original file line number	Diff line number	Diff line change
`@@ -111,6 +111,7 @@ class Meta:`
`111`	`111`	`"sync_parent_group",`
`112`	`112`	`"connectivity",`
`113`	`113`	`"lookup_groups_from_user",`
	`114`	`+ "delete_not_found_objects",`
`114`	`115`	`]`
`115`	`116`	`extra_kwargs = {"bind_password": {"write_only": True}}`
`116`	`117`
`@@ -147,6 +148,7 @@ class LDAPSourceViewSet(UsedByMixin, ModelViewSet):`
`147`	`148`	`"user_property_mappings",`
`148`	`149`	`"group_property_mappings",`
`149`	`150`	`"lookup_groups_from_user",`
	`151`	`+ "delete_not_found_objects",`
`150`	`152`	`]`
`151`	`153`	`search_fields = ["name", "slug"]`
`152`	`154`	`ordering = ["name"]`