MSC3911: Handle media that are attached to redacted media

synapse/api/auth/base.py

@@ -239,6 +239,51 @@ async def check_can_change_room_list(
 
         return user_level >= send_level
 
+    async def is_moderator(self, room_id: str, requester: Requester) -> bool:
+        """Determine whether the user is moderator of the room.
+
+        Args:
+            room_id: The room_id of the room to check
+            requester: The user making the request
+        """
+        is_admin = await self.is_server_admin(requester)
+        if is_admin:
+            return True
+        await self.check_user_in_room(room_id, requester)
+
+        # We currently require the user is a "moderator" in the room. We do this
+        # by checking if they would (theoretically) be able to change the
+        # m.room.canonical_alias events
+
+        auth_events = await self._storage_controllers.state.get_current_state(
+            room_id,
+            StateFilter.from_types(
+                [
+                    POWER_KEY,
+                    CREATE_KEY,
+                ]
+            ),
+        )
+
+        send_level = event_auth.get_send_level(
+            EventTypes.CanonicalAlias,
+            "",
+            auth_events.get(POWER_KEY),
+        )
+
+        user_level = event_auth.get_user_power_level(
+            requester.user.to_string(), auth_events
+        )
+        # Check multiple moderator-level actions
+        kick_level = event_auth.get_named_level(auth_events, "kick", 50)
+        ban_level = event_auth.get_named_level(auth_events, "ban", 50)
+        redact_level = event_auth.get_named_level(auth_events, "redact", 50)
+
+        # Consider someone a moderator if they can perform key mod actions
+        moderator_threshold = min(kick_level, ban_level, redact_level)
+
+        return user_level >= send_level and user_level >= moderator_threshold
+
     @staticmethod
     def has_access_token(request: Request) -> bool:
         """Checks if the request has an access_token.

synapse/config/experimental.py

@@ -592,7 +592,12 @@ def read_config(
         # MSC3911: Linking Media to Events
         self.msc3911_enabled: bool = experimental.get("msc3911_enabled", False)
 
-        # Disable the current media create and upload endpoints
+        # MSC3911: Disable the current media create and upload endpoints
         self.msc3911_unrestricted_media_upload_disabled: bool = experimental.get(
             "msc3911_unrestricted_media_upload_disabled", False
         )
+
+        # MSC3911: Retention time for media that are attached to redacted events
+        self.msc3911_redacted_event_media_cleanup_interval: int = experimental.get(
+            "msc3911_redacted_event_media_cleanup_interval", 48 * 60 * 60 * 1000
+        )

synapse/handlers/events.py

@@ -156,14 +156,17 @@ async def get_event(
         event_id: str,
         show_redacted: bool = False,
     ) -> Optional[EventBase]:
-        """Retrieve a single specified event.
+        """Retrieve a single specified event from the database, depending on the
+        show_redacted flag, it either hides or exposes redacted content.
 
         Args:
             user: The local user requesting the event
             room_id: The expected room id. We'll return None if the
                 event's room does not match.
             event_id: The event ID to obtain.
-            show_redacted: Should the full content of redacted events be returned?
+            show_redacted: If False (default), the returned event will have its redacted
+                content removed as users normally see it. When True, the event will be
+                returned exactly as stored, even if it’s been redacted.
         Returns:
             An event, or None if there is no event matching this ID.
         Raises:

synapse/media/media_repository.py

@@ -33,6 +33,7 @@
 import twisted.web.http
 from twisted.internet.defer import Deferred
 
+from synapse.api.auth.base import BaseAuth
 from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import (
     Codes,
@@ -213,7 +214,7 @@ async def delete_old_remote_media(self, before_ts: int) -> Dict[str, int]:
             "Sorry Mario, your MediaRepository related function is in another castle"
         )
 
-    async def delete_local_media_ids(
+    async def delete_media_from_disk_by_media_ids(
         self, media_ids: List[str]
     ) -> Tuple[List[str], int]:
         raise NotImplementedError(
@@ -312,13 +313,13 @@ async def validate_media_restriction(
         return attachments
 
     async def is_media_visible(
-        self, requesting_user: UserID, media_info_object: Union[LocalMedia, RemoteMedia]
+        self, requester: Requester, media_info_object: Union[LocalMedia, RemoteMedia]
     ) -> None:
         """
         Verify that media requested for download should be visible to the user making
         the request
         """
-
+        requester_user_id_str = requester.user.to_string()
         if not self.enable_media_restriction:
             return
 
@@ -329,7 +330,7 @@ async def is_media_visible(
             # When the media has not been attached yet, only the originating user can
             # see it. But once attachments have been formed, standard other rules apply
             if isinstance(media_info_object, LocalMedia) and (
-                requesting_user.to_string() == str(media_info_object.user_id)
+                requester_user_id_str == media_info_object.user_id
             ):
                 return
 
@@ -338,7 +339,7 @@ async def is_media_visible(
                 "Media ID ('%s') as requested by '%s' was restricted but had no "
                 "attachments",
                 media_info_object.media_id,
-                requesting_user.to_string(),
+                requester_user_id_str,
             )
             raise UnauthorizedRequestAPICallError(
                 f"Media requested ('{media_info_object.media_id}') is restricted"
@@ -348,7 +349,29 @@ async def is_media_visible(
         attached_profile_user_id = media_info_object.attachments.profile_user_id
 
         if attached_event_id:
+            # Check if event is redacted or not. If it is redacted, normal user no
+            # longer have access to the media, but only moderators can see the media
+            # until it gets deleted permanently after certain period of time.
             event_base = await self.store.get_event(attached_event_id)
+            if event_base.internal_metadata.is_redacted():
+                # In the case of redacted media, check the requester's power level of the room
+                room_id = event_base.room_id
+                assert isinstance(self.auth, BaseAuth)
+                is_moderator = await self.auth.is_moderator(room_id, requester)
+                if is_moderator:
+                    # Moderator still has access to the media that is attached to redacted event
+                    return
+                else:
+                    logger.debug(
+                        "Media ID (%s) as requested by '%s' was redacted from event '%s'",
+                        media_info_object.media_id,
+                        requester_user_id_str,
+                        attached_event_id,
+                    )
+                    raise UnauthorizedRequestAPICallError(
+                        f"Media requested ('{media_info_object.media_id}') is restricted"
+                    )
+
             if event_base.is_state():
                 # The standard event visibility utility, filter_events_for_client(),
                 # does not seem to meet the needs of a good UX when restricting and
@@ -368,13 +391,13 @@ async def is_media_visible(
                     membership_now,
                     _,
                 ) = await self.store.get_local_current_membership_for_user_in_room(
-                    requesting_user.to_string(), event_base.room_id
+                    requester_user_id_str, event_base.room_id
                 )
 
                 if not membership_now:
                     membership_now = Membership.LEAVE
 
-                membership_state_key = (EventTypes.Member, requesting_user.to_string())
+                membership_state_key = (EventTypes.Member, requester_user_id_str)
                 types = (_HISTORY_VIS_KEY, membership_state_key)
 
                 # and history visibility and membership of THEN
@@ -467,7 +490,7 @@ async def is_media_visible(
                 storage_controllers = self.hs.get_storage_controllers()
                 filtered_events = await filter_events_for_client(
                     storage_controllers,
-                    requesting_user.to_string(),
+                    requester_user_id_str,
                     [event_base],
                 )
                 if len(filtered_events) > 0:
@@ -487,22 +510,22 @@ async def is_media_visible(
             if self.hs.config.server.limit_profile_requests_to_users_who_share_rooms:
                 # First take care of the case where the requesting user IS the creating
                 # user. The other function below does not handle this.
-                if requesting_user.to_string() == attached_profile_user_id.to_string():
+                if requester_user_id_str == attached_profile_user_id:
                     return
 
                 # This call returns a set() that contains which of the "other_user_ids"
                 # share a room. Since we give it only one, if bool(set()) is True, then they
                 # share some room or had at least one invite between them.
                 if not await self.store.do_users_share_a_room_joined_or_invited(
-                    requesting_user.to_string(),
-                    [attached_profile_user_id.to_string()],
+                    requester_user_id_str,
+                    [attached_profile_user_id],
                 ):
                     logger.debug(
                         "Media ID (%s) as requested by '%s' was restricted by "
                         "profile, but was not allowed(is "
                         "'limit_profile_requests_to_users_who_share_rooms' enabled?)",
                         media_info_object.media_id,
-                        requesting_user.to_string(),
+                        requester_user_id_str,
                     )
 
                     raise UnauthorizedRequestAPICallError(
@@ -521,7 +544,7 @@ async def is_media_visible(
             "Media ID (%s) as requested by '%s' was restricted, but was not "
             "allowed(media_attachments=%s)",
             media_info_object.media_id,
-            requesting_user.to_string(),
+            requester_user_id_str,
             media_info_object.attachments,
         )
         raise UnauthorizedRequestAPICallError(
@@ -609,6 +632,13 @@ def __init__(self, hs: "HomeServer"):
             self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS
         )
 
+        self.clock.looping_call(
+            self._redacted_media_cleanup,
+            float(
+                self.hs.config.experimental.msc3911_redacted_event_media_cleanup_interval
+            ),
+        )
+
         # Media retention configuration options
         self._media_retention_local_media_lifetime_ms = (
             hs.config.media.media_retention_local_media_lifetime_ms
@@ -949,7 +979,7 @@ async def get_local_media_info(
             # The file has been uploaded, so stop looping
             if media_info.media_length is not None:
                 if isinstance(request.requester, Requester):
-                    await self.is_media_visible(request.requester.user, media_info)
+                    await self.is_media_visible(request.requester, media_info)
                 return media_info
 
             # Check if the media ID has expired and still hasn't been uploaded to.
@@ -1012,7 +1042,7 @@ async def get_local_media(
             if requester is not None:
                 # Only check media visibility if this is for a local request. This will
                 # raise directly back to the client if not visible
-                await self.is_media_visible(requester.user, media_info)
+                await self.is_media_visible(requester, media_info)
             restrictions = await self.validate_media_restriction(
                 request, media_info, None, federation
             )
@@ -1242,7 +1272,7 @@ async def _get_remote_media_impl(
             # retrieved from the remote.
             if self.enable_media_restriction and requester is not None:
                 # This will raise directly back to the client if not visible
-                await self.is_media_visible(requester.user, media_info)
+                await self.is_media_visible(requester, media_info)
 
             # file_id is the ID we use to track the file locally. If we've already
             # seen the file then reuse the existing ID, otherwise generate a new
@@ -1300,7 +1330,7 @@ async def _get_remote_media_impl(
             and requester is not None
         ):
             # This will raise directly back to the client if not visible
-            await self.is_media_visible(requester.user, media_info)
+            await self.is_media_visible(requester, media_info)
 
         file_id = media_info.filesystem_id
         if not media_info.media_type:
@@ -2067,7 +2097,7 @@ async def delete_old_remote_media(self, before_ts: int) -> Dict[str, int]:
 
         return {"deleted": deleted}
 
-    async def delete_local_media_ids(
+    async def delete_media_from_disk_by_media_ids(
         self, media_ids: List[str]
     ) -> Tuple[List[str], int]:
         """
@@ -2078,7 +2108,7 @@ async def delete_local_media_ids(
         Returns:
             A tuple of (list of deleted media IDs, total deleted media IDs).
         """
-        return await self._remove_local_media_from_disk(media_ids)
+        return await self._remove_media_from_disk(media_ids)
 
     async def delete_old_local_media(
         self,
@@ -2111,9 +2141,9 @@ async def delete_old_local_media(
             include_quarantined_media=delete_quarantined_media,
             include_protected_media=delete_protected_media,
         )
-        return await self._remove_local_media_from_disk(old_media)
+        return await self._remove_media_from_disk(old_media)
 
-    async def _remove_local_media_from_disk(
+    async def _remove_media_from_disk(
         self, media_ids: List[str]
     ) -> Tuple[List[str], int]:
         """
@@ -2149,3 +2179,17 @@ async def _remove_local_media_from_disk(
             removed_media.append(media_id)
 
         return removed_media, len(removed_media)
+
+    async def _redacted_media_cleanup(self) -> None:
+        """Periodically deletes media attached to redacted events from disk."""
+        redacted_event_ids = await self.store.get_redacted_event_ids_before_interval(
+            self.hs.config.experimental.msc3911_redacted_event_media_cleanup_interval
+        )
+        media_ids_to_redact = []
+        for event_id in redacted_event_ids:
+            attached_media_ids = await self.store.get_media_ids_attached_to_event(
+                event_id
+            )
+            media_ids_to_redact.extend(attached_media_ids)
+        if media_ids_to_redact:
+            await self.delete_media_from_disk_by_media_ids(media_ids_to_redact)

synapse/media/thumbnailer.py

@@ -305,7 +305,7 @@ async def respond_local_thumbnail(
             if requester is not None:
                 # Only check media visibility if this is for a local request. This will
                 # raise directly back to the client if not visible
-                await self.media_repo.is_media_visible(requester.user, media_info)
+                await self.media_repo.is_media_visible(requester, media_info)
             restrictions = await self.media_repo.validate_media_restriction(
                 request, media_info, None, for_federation
             )
@@ -365,7 +365,7 @@ async def select_or_generate_local_thumbnail(
             if requester is not None:
                 # Only check media visibility if this is for a local request. This will
                 # raise directly back to the client if not visible
-                await self.media_repo.is_media_visible(requester.user, media_info)
+                await self.media_repo.is_media_visible(requester, media_info)
             restrictions = await self.media_repo.validate_media_restriction(
                 request, None, media_id, for_federation
             )
@@ -482,7 +482,7 @@ async def select_or_generate_remote_thumbnail(
         # if MSC3911 is enabled, check visibility of the media for the user
         if self.enable_media_restriction and requester is not None:
             # This will raise directly back to the client if not visible
-            await self.media_repo.is_media_visible(requester.user, media_info)
+            await self.media_repo.is_media_visible(requester, media_info)
 
         # Check if the media is cached on the client, if so return 304.
         if check_for_cached_entry_and_respond(request):
@@ -572,7 +572,7 @@ async def respond_remote_thumbnail(
         # if MSC3911 is enabled, check visibility of the media for the user
         if self.enable_media_restriction and requester is not None:
             # This will raise directly back to the client if not visible
-            await self.media_repo.is_media_visible(requester.user, media_info)
+            await self.media_repo.is_media_visible(requester, media_info)
 
         # Check if the media is cached on the client, if so return 304.
         if check_for_cached_entry_and_respond(request):

synapse/rest/admin/media.py

@@ -282,9 +282,10 @@ async def on_DELETE(
 
         logger.info("Deleting local media by ID: %s", media_id)
 
-        deleted_media, total = await self.media_repository.delete_local_media_ids(
-            [media_id]
-        )
+        (
+            deleted_media,
+            total,
+        ) = await self.media_repository.delete_media_from_disk_by_media_ids([media_id])
         return HTTPStatus.OK, {"deleted_media": deleted_media, "total": total}
 
 
@@ -446,7 +447,10 @@ async def on_DELETE(
             start, limit, user_id, order_by, direction
         )
 
-        deleted_media, total = await self.media_repository.delete_local_media_ids(
+        (
+            deleted_media,
+            total,
+        ) = await self.media_repository.delete_media_from_disk_by_media_ids(
             [m.media_id for m in media]
         )