Transform events with client metadata before serialising in /event response. (#19340)

Fix /event/ endpoint not transforming event with per-requester metadata 

Pass notif_event through filter_events_for_client \
Not aware of an actual issue here, but seems silly to bypass it

Call it filter_and_transform_events_for_client to make it more obvious 

---------

Signed-off-by: Olivier 'reivilibre <oliverw@matrix.org>

changelog.d/19340.bugfix

@@ -0,0 +1 @@
+Transform events with client metadata before serialising in /event response.

synapse/handlers/admin.py

@@ -44,7 +44,7 @@ from synapse.types import (
     UserInfo,
     create_requester,
 )
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -251,7 +251,7 @@ class AdminHandler:
                     topological=last_event.depth,
                 )
 
-                events = await filter_events_for_client(
+                events = await filter_and_transform_events_for_client(
                     self._storage_controllers,
                     user_id,
                     events,

synapse/handlers/events.py

@@ -31,7 +31,7 @@ from synapse.handlers.presence import format_user_presence_state
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, Requester, UserID
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -156,7 +156,9 @@ class EventHandler:
         event_id: str,
         show_redacted: bool = False,
     ) -> EventBase | None:
-        """Retrieve a single specified event.
+        """Retrieve a single specified event on behalf of a user.
+        The event will be transformed in a user-specific and time-specific way,
+        e.g. having unsigned metadata added or being erased depending on who is accessing.
 
         Args:
             user: The local user requesting the event
@@ -188,7 +190,7 @@ class EventHandler:
         # The user is peeking if they aren't in the room already
         is_peeking = not is_user_in_room
 
-        filtered = await filter_events_for_client(
+        filtered = await filter_and_transform_events_for_client(
             self._storage_controllers,
             user.to_string(),
             [event],
@@ -198,4 +200,4 @@ class EventHandler:
         if not filtered:
             raise AuthError(403, "You don't have permission to access that event.")
 
-        return event
+        return filtered[0]

synapse/handlers/initial_sync.py

@@ -49,7 +49,7 @@ from synapse.types import (
 from synapse.util import unwrapFirstError
 from synapse.util.async_helpers import concurrently_execute, gather_results
 from synapse.util.caches.response_cache import ResponseCache
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -225,7 +225,7 @@ class InitialSyncHandler:
                     )
                 ).addErrback(unwrapFirstError)
 
-                messages = await filter_events_for_client(
+                messages = await filter_and_transform_events_for_client(
                     self._storage_controllers,
                     user_id,
                     messages,
@@ -382,7 +382,7 @@ class InitialSyncHandler:
             room_id, limit=pagin_config.limit, end_token=stream_token
         )
 
-        messages = await filter_events_for_client(
+        messages = await filter_and_transform_events_for_client(
             self._storage_controllers,
             requester.user.to_string(),
             messages,
@@ -496,7 +496,7 @@ class InitialSyncHandler:
             ).addErrback(unwrapFirstError)
         )
 
-        messages = await filter_events_for_client(
+        messages = await filter_and_transform_events_for_client(
             self._storage_controllers,
             requester.user.to_string(),
             messages,

synapse/handlers/pagination.py

@@ -46,7 +46,7 @@ from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse
 from synapse.types.state import StateFilter
 from synapse.util.async_helpers import ReadWriteLock
 from synapse.util.duration import Duration
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -684,7 +684,7 @@ class PaginationHandler:
             events = await event_filter.filter(events)
 
         if not use_admin_priviledge:
-            events = await filter_events_for_client(
+            events = await filter_and_transform_events_for_client(
                 self._storage_controllers,
                 user_id,
                 events,

synapse/handlers/relations.py

@@ -40,7 +40,7 @@ from synapse.storage.databases.main.relations import ThreadsNextBatch, _RelatedE
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, Requester, UserID
 from synapse.util.async_helpers import gather_results
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -154,7 +154,7 @@ class RelationsHandler:
             [e.event_id for e in related_events]
         )
 
-        events = await filter_events_for_client(
+        events = await filter_and_transform_events_for_client(
             self._storage_controllers,
             user_id,
             events,
@@ -599,7 +599,7 @@ class RelationsHandler:
             # Limit the returned threads to those the user has participated in.
             events = [event for event in events if participated[event.event_id]]
 
-        events = await filter_events_for_client(
+        events = await filter_and_transform_events_for_client(
             self._storage_controllers,
             user_id,
             events,

synapse/handlers/room.py

@@ -95,7 +95,7 @@ from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.duration import Duration
 from synapse.util.iterutils import batch_iter
 from synapse.util.stringutils import parse_and_validate_server_name
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -1919,7 +1919,7 @@ class RoomContextHandler:
         async def filter_evts(events: list[EventBase]) -> list[EventBase]:
             if use_admin_priviledge:
                 return events
-            return await filter_events_for_client(
+            return await filter_and_transform_events_for_client(
                 self._storage_controllers,
                 user.to_string(),
                 events,

synapse/handlers/search.py

@@ -33,7 +33,7 @@ from synapse.events import EventBase
 from synapse.events.utils import SerializeEventConfig
 from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID
 from synapse.types.state import StateFilter
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -479,7 +479,7 @@ class SearchHandler:
 
         filtered_events = await search_filter.filter([r["event"] for r in results])
 
-        events = await filter_events_for_client(
+        events = await filter_and_transform_events_for_client(
             self._storage_controllers,
             user.to_string(),
             filtered_events,
@@ -580,7 +580,7 @@ class SearchHandler:
 
             filtered_events = await search_filter.filter([r["event"] for r in results])
 
-            events = await filter_events_for_client(
+            events = await filter_and_transform_events_for_client(
                 self._storage_controllers,
                 user.to_string(),
                 filtered_events,
@@ -667,13 +667,13 @@ class SearchHandler:
                 len(res.events_after),
             )
 
-            events_before = await filter_events_for_client(
+            events_before = await filter_and_transform_events_for_client(
                 self._storage_controllers,
                 user.to_string(),
                 res.events_before,
             )
 
-            events_after = await filter_events_for_client(
+            events_after = await filter_and_transform_events_for_client(
                 self._storage_controllers,
                 user.to_string(),
                 res.events_after,

synapse/handlers/sliding_sync/__init__.py

@@ -71,7 +71,7 @@ from synapse.types.handlers.sliding_sync import (
 )
 from synapse.types.state import StateFilter
 from synapse.util.async_helpers import concurrently_execute
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -755,7 +755,7 @@ class SlidingSyncHandler:
             timeline_events.reverse()
 
             # Make sure we don't expose any events that the client shouldn't see
-            timeline_events = await filter_events_for_client(
+            timeline_events = await filter_and_transform_events_for_client(
                 self.storage_controllers,
                 user.to_string(),
                 timeline_events,

synapse/handlers/sync.py

@@ -78,7 +78,7 @@ from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.response_cache import ResponseCache, ResponseCacheContext
 from synapse.util.metrics import Measure
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -679,7 +679,7 @@ class SyncHandler:
                         )
                     )
 
-                recents = await filter_events_for_client(
+                recents = await filter_and_transform_events_for_client(
                     self._storage_controllers,
                     sync_config.user.to_string(),
                     recents,
@@ -789,7 +789,7 @@ class SyncHandler:
                         )
                     )
 
-                loaded_recents = await filter_events_for_client(
+                loaded_recents = await filter_and_transform_events_for_client(
                     self._storage_controllers,
                     sync_config.user.to_string(),
                     loaded_recents,

synapse/notifier.py

@@ -63,7 +63,7 @@ from synapse.util.async_helpers import (
 )
 from synapse.util.duration import Duration
 from synapse.util.stringutils import shortstr
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -783,7 +783,7 @@ class Notifier:
                 )
 
                 if keyname == StreamKeyType.ROOM:
-                    new_events = await filter_events_for_client(
+                    new_events = await filter_and_transform_events_for_client(
                         self._storage_controllers,
                         user.to_string(),
                         new_events,

synapse/push/mailer.py

@@ -49,7 +49,7 @@ from synapse.storage.databases.main.event_push_actions import EmailPushAction
 from synapse.types import StateMap, UserID
 from synapse.types.state import StateFilter
 from synapse.util.async_helpers import concurrently_execute
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -537,12 +537,11 @@ class Mailer:
             "messages": [],
         }
 
-        the_events = await filter_events_for_client(
+        the_events = await filter_and_transform_events_for_client(
             self._storage_controllers,
             user_id,
-            results.events_before,
+            results.events_before + [notif_event],
         )
-        the_events.append(notif_event)
 
         for event in the_events:
             messagevars = await self._get_message_vars(notif, event, room_state_ids)

synapse/visibility.py

@@ -75,7 +75,7 @@ _HISTORY_VIS_KEY: Final[tuple[str, str]] = (EventTypes.RoomHistoryVisibility, ""
 
 
 @trace
-async def filter_events_for_client(
+async def filter_and_transform_events_for_client(
     storage: StorageControllers,
     user_id: str,
     events: list[EventBase],

tests/rest/client/test_retention.py

@@ -28,7 +28,7 @@ from synapse.rest.client import login, room
 from synapse.server import HomeServer
 from synapse.types import JsonDict, create_requester
 from synapse.util.clock import Clock
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_and_transform_events_for_client
 
 from tests import unittest
 from tests.unittest import override_config
@@ -163,7 +163,7 @@ class RetentionTestCase(unittest.HomeserverTestCase):
         )
         self.assertEqual(2, len(events), "events retrieved from database")
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 storage_controllers,
                 self.user_id,
                 events,

tests/test_visibility.py

@@ -31,7 +31,10 @@ from synapse.rest.client import login, room
 from synapse.server import HomeServer
 from synapse.types import create_requester
 from synapse.util.clock import Clock
-from synapse.visibility import filter_events_for_client, filter_events_for_server
+from synapse.visibility import (
+    filter_and_transform_events_for_client,
+    filter_events_for_server,
+)
 
 from tests import unittest
 from tests.test_utils.event_injection import inject_event, inject_member_event
@@ -330,7 +333,7 @@ class FilterEventsForServerAdminsTestCase(HomeserverTestCase):
 
         # Do filter & assert
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@admin:test",
                 events_to_filter,
@@ -369,7 +372,7 @@ class FilterEventsForServerAdminsTestCase(HomeserverTestCase):
 
         # Do filter & assert
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@admin:test",
                 events_to_filter,
@@ -416,7 +419,7 @@ class FilterEventsForServerAdminsTestCase(HomeserverTestCase):
 
         # Do filter & assert
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@admin:test",
                 events_to_filter,
@@ -463,7 +466,7 @@ class FilterEventsForServerAdminsTestCase(HomeserverTestCase):
 
         # Do filter & assert
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@admin:test",
                 events_to_filter,
@@ -538,14 +541,14 @@ class FilterEventsForClientTestCase(HomeserverTestCase):
         # accidentally serving the same event object (with the same unsigned.membership
         # property) to both users.
         joiner_filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@joiner:test",
                 events_to_filter,
             )
         )
         resident_filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@resident:test",
                 events_to_filter,
@@ -641,7 +644,7 @@ class FilterEventsOutOfBandEventsForClientTestCase(
 
         # the invited user should be able to see both the invite and the rejection
         filtered_events = self.get_success(
-            filter_events_for_client(
+            filter_and_transform_events_for_client(
                 self.hs.get_storage_controllers(),
                 "@user:test",
                 [invite_event, reject_event],
@@ -662,7 +665,7 @@ class FilterEventsOutOfBandEventsForClientTestCase(
         # other users should see neither
         self.assertEqual(
             self.get_success(
-                filter_events_for_client(
+                filter_and_transform_events_for_client(
                     self.hs.get_storage_controllers(),
                     "@other:test",
                     [invite_event, reject_event],