diff --git a/flake.nix b/flake.nix index c15d2ba93..c8ef7fa74 100644 --- a/flake.nix +++ b/flake.nix @@ -61,7 +61,7 @@ ; pname = "sable"; fetcherVersion = 3; - hash = "sha256-LdNWHVhUur064v6Nd+c7qENg7f7k0067Ty8tqBvDMys="; + hash = "sha256-WIZM9ZHMtBIVEy6q1vP6WvOtnfW7V/Dokb7wUR8TKWc="; }; mkPnpmCheck = diff --git a/package.json b/package.json index 452bf7f96..36ddde781 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "emojibase-data": "^17.0.0", "eventemitter3": "^5.0.4", "file-saver": "^2.0.5", + "flexsearch": "0.8.212", "focus-trap-react": "^10.3.1", "folds": "^2.6.2", "framer-motion": "^12.40.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d39187a3c..0533d2c89 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,6 +98,9 @@ importers: file-saver: specifier: ^2.0.5 version: 2.0.5 + flexsearch: + specifier: 0.8.212 + version: 0.8.212 focus-trap-react: specifier: ^10.3.1 version: 10.3.1(prop-types@15.8.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -3524,6 +3527,9 @@ packages: flatted@3.4.2: resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + flexsearch@0.8.212: + resolution: {integrity: sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw==} + focus-trap-react@10.3.1: resolution: {integrity: sha512-PN4Ya9xf9nyj/Nd9VxBNMuD7IrlRbmaG6POAQ8VLqgtc6IY/Ln1tYakow+UIq4fihYYYFM70/2oyidE6bbiPgw==} peerDependencies: @@ -8199,6 +8205,8 @@ snapshots: flatted@3.4.2: {} + flexsearch@0.8.212: {} + focus-trap-react@10.3.1(prop-types@15.8.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: focus-trap: 7.8.0 diff --git a/src/app/components/GlobalKeyboardShortcuts.tsx b/src/app/components/GlobalKeyboardShortcuts.tsx index 4f2e4cf49..94e5c0219 100644 --- a/src/app/components/GlobalKeyboardShortcuts.tsx +++ b/src/app/components/GlobalKeyboardShortcuts.tsx @@ -20,6 +20,7 @@ import { getDirectRoomPath, getHomeRoomPath, getHomeSearchPath, + getDirectSearchPath, getSpaceRoomPath, getSpaceSearchPath, withSearchParam, @@ -31,6 +32,7 @@ import { announce } from '$utils/announce'; import { roomIdToReplyDraftAtomFamily } from '$state/room/roomInputDrafts'; import type { Room } from '$types/matrix-sdk'; import { useSelectedSpace } from '$hooks/router/useSelectedSpace'; +import { useIsDirectRoom } from '$hooks/useRoom'; export function GlobalKeyboardShortcuts() { const navigate = useNavigate(); @@ -38,6 +40,7 @@ export function GlobalKeyboardShortcuts() { const mx = useMatrixClient(); const roomToParents = useAtomValue(roomToParentsAtom); const mDirects = useAtomValue(mDirectAtom); + const direct = useIsDirectRoom(); const roomToUnread = useAtomValue(roomToUnreadAtom); const unreadIndexRef = useRef(0); @@ -173,7 +176,29 @@ export function GlobalKeyboardShortcuts() { }; const path = currentSpace ? getSpaceSearchPath(getCanonicalAliasOrRoomId(mx, currentSpace)) - : getHomeSearchPath(); + : direct + ? getDirectSearchPath() + : getHomeSearchPath(); + const roomName = mx.getRoom(currentRoom?.roomId)?.name; + navigate(withSearchParam(path, searchParams)); + announce(`Start Searching messages ${roomName ? `in ${roomName}` : ''}`); + }, + [mx, currentRoom, currentSpace, navigate] + ); + + const handleSearchMessageGlobally = useCallback( + (evt: KeyboardEvent) => { + if (!isKeyHotkey('mod+shift+f', evt)) return; + evt.preventDefault(); + + const searchParams: SearchPathSearchParams = { + global: 'true', + }; + const path = currentSpace + ? getSpaceSearchPath(getCanonicalAliasOrRoomId(mx, currentSpace)) + : direct + ? getDirectSearchPath() + : getHomeSearchPath(); const roomName = mx.getRoom(currentRoom?.roomId)?.name; navigate(withSearchParam(path, searchParams)); announce(`Start Searching messages ${roomName ? `in ${roomName}` : ''}`); @@ -185,6 +210,7 @@ export function GlobalKeyboardShortcuts() { useKeyDown(window, handleUnreadNavKeyDown); useKeyDown(window, handleReplyKeyDown); useKeyDown(window, handleSearchMessageInRoom); + useKeyDown(window, handleSearchMessageGlobally); return null; } diff --git a/src/app/components/SearchIndexProvider.tsx b/src/app/components/SearchIndexProvider.tsx new file mode 100644 index 000000000..4cd971fcc --- /dev/null +++ b/src/app/components/SearchIndexProvider.tsx @@ -0,0 +1,593 @@ +import { useMatrixClient } from '$hooks/useMatrixClient'; +import { + SearchIndexContext, + SearchIndexContextType, + SearchIndexState, +} from '$hooks/useSearchIndex'; +import { + SearchIndexEvent, + IndexWorkerMessageIn, + WorkerMessageTypeIn, + IndexWorkerMessageOut, + WorkerMessageTypeOut, + BackfillState, +} from '$plugins/search-indexer/types'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { + MatrixEvent, + EventType, + MatrixEventEvent, + Room, + EventTimelineSet, + Direction, + ClientEvent, + RoomEvent, + SyncState, +} from 'matrix-js-sdk'; +import { ReactNode, useState, useRef, useCallback, useMemo, useEffect } from 'react'; +import * as Sentry from '@sentry/react'; +import CreateSearchWorker from '$plugins/search-indexer/index.ts?worker'; +import { is } from 'immer/dist/internal.js'; + +const BACKFILL_PAGE_SIZE = 50; +const HAS_IDLE_CALLBACK = typeof requestIdleCallback === 'function'; +let IDLE_CALLBACK_COUNT = 0; +const MAX_CONCURRENT_BACKFILLS = HAS_IDLE_CALLBACK ? Infinity : 2; +const BACKFILL_STARTUP_DELAY_MS = 30_000; + +const canRunMobileBackfill = (): boolean => + HAS_IDLE_CALLBACK || (document.visibilityState === 'visible' && document.hasFocus()); + +function scheduleIdle(cb: () => void): () => void { + if (HAS_IDLE_CALLBACK) { + IDLE_CALLBACK_COUNT += 1; + const id = requestIdleCallback(cb, { timeout: 5000 }); + return () => cancelIdleCallback(id); + } + + const id = setTimeout(cb, 500); + return () => clearTimeout(id); +} + +const MEDIA_MSGTYPES = ['m.image', 'm.file', 'm.audio', 'm.video']; + +function toSearchIndexEvent(event: MatrixEvent, replaced: boolean = false): SearchIndexEvent | null { + const eventId = event.getId(); + if (!eventId) return null; + + const roomId = event.getRoomId(); + if (!roomId) return null; + + if (event.getType() !== EventType.RoomMessage) return null; + if (event.isRedacted()) return null; + + const content = event.getContent(); + + const body: string = (replaced ? event.getContent()["m.new_content"]?.body : content.body) ?? ''; + if (!body.trim()) return null; + + const sender = event.getSender(); + if (!sender) return null; + + const msgtype = content.msgtype ?? 'm.text'; + const ts = event.getTs(); + const hasLink = /https?:\/\//i.test(body) + + const searchIndexEvent: SearchIndexEvent = { eventId, roomId, sender, msgtype, body, ts, hasLink }; + + if (MEDIA_MSGTYPES.includes(msgtype)) { + if (content.url !== undefined) searchIndexEvent.url = content.url; + if (content.file !== undefined) searchIndexEvent.file = content.file; + if (content.info !== undefined) searchIndexEvent.info = content.info; + if (content.filename !== undefined) searchIndexEvent.filename = content.filename; + } + return searchIndexEvent; +} + +type PendingStats = { + resolve: (stats: SearchIndexState) => void; + backfillingRoomCount: number; +}; +type PendingQuery = { + resolve: (events: SearchIndexEvent[]) => void; + reject: (err: unknown) => void; +}; + +export function SearchIndexProvider({ children }: { children: ReactNode }) { + const mx = useMatrixClient(); + const [idbSearchIndex] = useSetting(settingsAtom, 'idbSearchIndex'); + + const [isReady, setIsReady] = useState(false); + const [isBackfilling, setIsBackfilling] = useState(false); + + const workerRef = useRef(null); + const pendingStatsRef = useRef(null); + const pendingQueriesRef = useRef>(new Map()); + + const postToWorker = useCallback((msg: IndexWorkerMessageIn) => { + workerRef.current?.postMessage(msg); + }, []); + + const headlessSetsRef = useRef>(new Map()); + const cancelIdlesRef = useRef void>>([]); + const syncStateRef = useRef(null); + + const backfillQueueRef = useRef>([]); + const backfillStatesRef = useRef>({}); + const backfillingRoomsRef = useRef>(new Set()); // TODO: I dont like this + const backfillStartDelayRef = useRef | null>(null); + const backfillReadyRef = useRef(false); + + const indexEvent = useCallback( + (event: MatrixEvent) => { + const handleEvent = () => { + const searchEvent = toSearchIndexEvent(event); + if (searchEvent) + postToWorker({ + type: WorkerMessageTypeIn.Index, + events: [searchEvent], + }); + }; + + if (event.getType() === 'm.room.encrypted') { + event.once(MatrixEventEvent.Decrypted, handleEvent); + } + else if (event.isSending()) { + event.once(MatrixEventEvent.LocalEventIdReplaced, handleEvent) + } + else { + handleEvent(); + } + }, + [postToWorker] + ); + + const backfillRoom = useCallback(async (room: Room, state: BackfillState) => { + if (state.done) return; + + if (!canRunMobileBackfill()) { + backfillingRoomsRef.current.delete(room.roomId); + backfillQueueRef.current.unshift({ room, state }); + Sentry.addBreadcrumb({ + category: 'search.backfill', + message: `Backfill deferred for room ${room.roomId}`, + data: { + reason: 'mobile_not_focused', + isMobile: !HAS_IDLE_CALLBACK, + visibilityState: document.visibilityState, + focused: document.hasFocus(), + }, + level: 'info', + }); + return; + } + + let headlessSet = headlessSetsRef.current.get(room.roomId); + if (!headlessSet) { + headlessSet = new EventTimelineSet(room, {}); + headlessSetsRef.current.set(room.roomId, headlessSet); + } + const headlessTimeline = headlessSet.getLiveTimeline(); + + const seedToken = state.token ?? room.getLiveTimeline().getPaginationToken(Direction.Backward); + if (!seedToken) { + postToWorker({ + type: WorkerMessageTypeIn.SetBackfillState, + roomId: room.roomId, + state: { ...state, done: true }, + }); + backfillingRoomsRef.current.delete(room.roomId); + return; + } + + headlessTimeline.setPaginationToken(seedToken, Direction.Backward); + + const prevEventCount = headlessTimeline.getEvents().length; + + let hasMore = false; + try { + hasMore = await mx.paginateEventTimeline(headlessTimeline, { + backwards: true, + limit: BACKFILL_PAGE_SIZE, + }); + } catch { + backfillingRoomsRef.current.delete(room.roomId); + } + + const allEvents = headlessTimeline.getEvents(); + const newEvents = allEvents.slice(0, allEvents.length - prevEventCount); + + const recoveryFrontier = state.token === null ? state.oldestTs : undefined; + const unindexedEvents = + recoveryFrontier !== undefined + ? newEvents.filter((ev) => ev.getTs() < recoveryFrontier) + : newEvents; + + const events: SearchIndexEvent[] = []; + for (const ev of unindexedEvents) { + try { + const relatesTo = ev.getContent()?.['m.relates_to']; + if (relatesTo?.rel_type === 'm.thread' || ev.isRedacted()) { + continue; + } + + if (ev.getType() === 'm.room.encrypted') { + indexEvent(ev); + } else { + const indexable = toSearchIndexEvent(ev); + if (indexable) events.push(indexable); + } + } catch (e) { + continue; + } + } + + if (events.length > 0) { + postToWorker({ type: WorkerMessageTypeIn.Index, events }); + } + + const nextToken = headlessTimeline.getPaginationToken(Direction.Backward); + const done = !hasMore && !nextToken; + + // Track the oldest event timestamp we've indexed so far. Only update when + // we actually processed new events (unindexedEvents may be empty on the + // first page of expired-token recovery while fast-forwarding the frontier). + const minTsThisPage = + unindexedEvents.length > 0 ? Math.min(...unindexedEvents.map((e) => e.getTs())) : undefined; + const newOldestTs = + minTsThisPage !== undefined + ? Math.min(state.oldestTs ?? Infinity, minTsThisPage) + : state.oldestTs; + + postToWorker({ + type: WorkerMessageTypeIn.SetBackfillState, + roomId: room.roomId, + state: { + token: nextToken, + done, + indexedCount: state.indexedCount + events.length, + oldestTs: newOldestTs, + }, + }); + + if (!done) { + // Schedule next page — but yield to the main sync if it's struggling. + // resumeBackfill will restart this room once sync recovers. + const nextState: BackfillState = { + token: nextToken, + done: false, + indexedCount: state.indexedCount + events.length, + oldestTs: newOldestTs, + }; + const cancel = scheduleIdle(() => { + const s = syncStateRef.current; + if (s !== SyncState.Syncing && s !== SyncState.Prepared && s !== SyncState.Catchup) { + backfillingRoomsRef.current.delete(room.roomId); + backfillQueueRef.current.unshift({ room, state: nextState }); + Sentry.addBreadcrumb({ + category: 'search.backfill', + message: `Backfill deferred for room ${room.roomId}`, + data: { reason: 'sync_not_healthy', syncState: s }, + level: 'info', + }); + return; + } + + if (!canRunMobileBackfill()) { + backfillingRoomsRef.current.delete(room.roomId); + backfillQueueRef.current.unshift({ room, state: nextState }); + Sentry.addBreadcrumb({ + category: 'search.backfill', + message: `Backfill deferred for room ${room.roomId}`, + data: { + reason: 'mobile_not_focused', + isMobile: !HAS_IDLE_CALLBACK, + visibilityState: document.visibilityState, + focused: document.hasFocus(), + }, + level: 'info', + }); + return; + } + void backfillRoom(room, nextState); + }); + cancelIdlesRef.current.push(cancel); + } else { + backfillingRoomsRef.current.delete(room.roomId); + // Dequeue the next room from the concurrency queue while under the limit + while ( + backfillingRoomsRef.current.size < MAX_CONCURRENT_BACKFILLS && + backfillQueueRef.current.length > 0 + ) { + const next = backfillQueueRef.current.shift()!; + backfillingRoomsRef.current.add(next.room.roomId); + const cancel = scheduleIdle(() => void backfillRoom(next.room, next.state)); + cancelIdlesRef.current.push(cancel); + } + if (backfillingRoomsRef.current.size === 0 && backfillQueueRef.current.length === 0) { + setIsBackfilling(false); + } + } + }, []); + + const resumeBackfill = useCallback(() => { + if (!backfillReadyRef.current) return; + const s = syncStateRef.current; + if (s !== SyncState.Syncing && s !== SyncState.Prepared && s !== SyncState.Catchup) return; + if (!canRunMobileBackfill()) return; + + while ( + backfillingRoomsRef.current.size < MAX_CONCURRENT_BACKFILLS && + backfillQueueRef.current.length > 0 + ) { + const next = backfillQueueRef.current.shift()!; + backfillingRoomsRef.current.add(next.room.roomId); + const cancel = scheduleIdle(() => void backfillRoom(next.room, next.state)); + cancelIdlesRef.current.push(cancel); + } + }, [backfillRoom]); + + const startBackfill = useCallback( + (backfillStates: Record) => { + backfillStatesRef.current = backfillStates; + const rooms = mx.getRooms().filter((r) => !r.isSpaceRoom()); + for (const room of rooms) { + const state = backfillStates[room.roomId] ?? { + token: null, + done: false, + indexedCount: 0, + }; + if (state.done) continue; + if (backfillingRoomsRef.current.has(room.roomId)) continue; + if (backfillQueueRef.current.some((e) => e.room.roomId === room.roomId)) continue; + + backfillQueueRef.current.push({ room, state }); + } + + if (backfillQueueRef.current.length > 0 || backfillingRoomsRef.current.size > 0) { + setIsBackfilling(true); + } + + backfillStartDelayRef.current = setTimeout(() => { + backfillStartDelayRef.current = null; + backfillReadyRef.current = true; + resumeBackfill(); + }, BACKFILL_STARTUP_DELAY_MS); + }, + [mx, resumeBackfill] + ); + + const handleWorkerMessage = useCallback((event: MessageEvent) => { + const msg = event.data; + + switch (msg.type) { + case WorkerMessageTypeOut.Ready: + setIsReady(true); + postToWorker({ type: WorkerMessageTypeIn.GetBackfillStates }); + break; + case WorkerMessageTypeOut.State: + const pending = pendingStatsRef.current; + if (pending) { + pendingStatsRef.current = null; + pending.resolve({ + indexedEventsCount: msg.indexedEventCount, + roomCount: msg.roomCount, + backfillingRoomCount: pending.backfillingRoomCount, + }); + } + break; + case WorkerMessageTypeOut.QueryResult: + const pendingQuery = pendingQueriesRef.current.get(msg.id); + if (pendingQuery) { + pendingQueriesRef.current.delete(msg.id); + pendingQuery.resolve(msg.events); + } + break; + case WorkerMessageTypeOut.BackfillStatesDone: + startBackfill(msg.states); + break; + } + }, []); + + useEffect(() => { + if (!idbSearchIndex) { + setIsReady(false); + return () => {}; + } + let worker: Worker; + try { + Sentry.addBreadcrumb({ + category: 'search.index', + message: 'Initializing search worker', + level: 'info', + }); + worker = new CreateSearchWorker(); + } catch (e) { + console.error('Error!'); + return () => {}; + } + + const userId = mx.getUserId(); + if (!userId) return () => {}; + + Sentry.addBreadcrumb({ + category: 'search.index', + message: 'Initializing search worker', + level: 'info', + data: { userId }, + }); + + workerRef.current = worker; + worker.addEventListener('message', handleWorkerMessage); + + Sentry.addBreadcrumb({ + category: 'search.index', + message: 'INIT sent to worker', + level: 'info', + data: { userId }, + }); + + postToWorker({ + type: WorkerMessageTypeIn.Init, + userId, + }); + + const handleSync = (state: SyncState) => { + syncStateRef.current = state; + if ( + state === SyncState.Syncing + ) { + resumeBackfill(); + } + }; + mx.on(ClientEvent.Sync, handleSync as unknown as (...args: unknown[]) => void); + + const handleTimeline = (mEvent: MatrixEvent, room: Room | undefined) => { + const relation = mEvent.getRelation(); + + if (relation && relation.rel_type === "m.replace") { + const targetEventId = relation.event_id; + if (!targetEventId) return; + const searchEvent = toSearchIndexEvent(mEvent, true); + if (!searchEvent) return; + + postToWorker({ + type: WorkerMessageTypeIn.EditEvents, + events: { + [targetEventId]: searchEvent + } + }) + return; + } + + if (!room) return; + indexEvent(mEvent); + }; + mx.on(RoomEvent.Timeline, handleTimeline); + + const handleRedaction = (mEvent: MatrixEvent) => { + if (mEvent.getType() !== EventType.RoomRedaction) return; + let eventId = mEvent.event.redacts; + if (!eventId) return; + + postToWorker({ + type: WorkerMessageTypeIn.RedactEvents, + eventIds: [eventId] + }) + }; + mx.on(RoomEvent.Redaction, handleRedaction); + + const handleRoomAdded = (room: Room) => { + if (room.isSpaceRoom()) return; + if (backfillingRoomsRef.current.has(room.roomId)) return; + if (backfillQueueRef.current.some((e) => e.room.roomId === room.roomId)) return; + const state = backfillStatesRef.current[room.roomId] ?? { + token: null, + done: false, + indexedCount: 0, + }; + if (state.done) return; + backfillQueueRef.current.push({ room, state }); + setIsBackfilling(true); + resumeBackfill(); + }; + mx.on(ClientEvent.Room, handleRoomAdded); + + const handleForegroundFocus = () => { + if (canRunMobileBackfill()) { + resumeBackfill(); + } + }; + if (!HAS_IDLE_CALLBACK) { + document.addEventListener('visibilitychange', handleForegroundFocus); + window.addEventListener('focus', handleForegroundFocus); + window.addEventListener('pageshow', handleForegroundFocus); + } + +const handleOnBeforeUnload = () => { + postToWorker({ type: WorkerMessageTypeIn.Flush }); + + worker.addEventListener( + 'message', + (ev: MessageEvent) => { + if (ev.data.type === WorkerMessageTypeOut.FlushDone) { + worker.removeEventListener('message', handleWorkerMessage); + worker.terminate(); + workerRef.current = null; + } + }, + { once: true } + ); +} + + window.addEventListener('beforeunload', handleOnBeforeUnload); + + return () => { + mx.removeListener(ClientEvent.Sync, handleSync); + mx.removeListener( + RoomEvent.Timeline, + handleTimeline + ); + mx.removeListener( + ClientEvent.Room, + handleRoomAdded + ); + mx.removeListener(RoomEvent.Redaction, handleRedaction) + worker.removeEventListener('message', handleWorkerMessage); + // worker.removeEventListener('error', handleWorkerError); + window.removeEventListener('beforeunload', handleOnBeforeUnload); + + setIsReady(false); + setIsBackfilling(false); + }; + }, [idbSearchIndex, mx, handleWorkerMessage, indexEvent, postToWorker]); + + const query = useCallback( + ( + term: string, + opts?: { roomIds?: string[]; senders?: string[]; hasTypes?: string[] } + ): Promise => { + if (!workerRef.current || !isReady) return Promise.resolve([]); + const id = crypto.randomUUID(); + return new Promise((resolve, reject) => { + pendingQueriesRef.current.set(id, { resolve, reject }); + postToWorker({ + type: WorkerMessageTypeIn.Query, + id, + term, + roomIds: opts?.roomIds, + senders: opts?.senders, + hasTypes: opts?.hasTypes, + }); + }); + }, + [isReady, postToWorker] + ); + + const state = useCallback((): Promise => { + if (!workerRef.current || !isReady) { + return Promise.resolve({ + indexedEventsCount: 0, + roomCount: 0, + backfillingRoomCount: 0, + }); + } + return new Promise((resolve) => { + pendingStatsRef.current = { + backfillingRoomCount: backfillingRoomsRef.current.size, + resolve, + }; + postToWorker({ type: WorkerMessageTypeIn.State }); + }); + }, [isReady, postToWorker]); + + const clearIndex = () => {}; + + const ctx = useMemo( + () => ({ query, state, clearIndex, isBackfilling, ready: isReady }), + [query, state, clearIndex, isReady, isBackfilling] + ); + + return {children}; +} diff --git a/src/app/features/message-search/MessageSearch.tsx b/src/app/features/message-search/MessageSearch.tsx index c18d1ba15..96c65f790 100644 --- a/src/app/features/message-search/MessageSearch.tsx +++ b/src/app/features/message-search/MessageSearch.tsx @@ -21,8 +21,8 @@ import { useRooms } from '$state/hooks/roomList'; import { allRoomsAtom } from '$state/room-list/roomList'; import { mDirectAtom } from '$state/mDirectList'; import { VirtualTile } from '$components/virtualizer'; -import type { MessageSearchParams } from './useMessageSearch'; -import { useMessageSearch } from './useMessageSearch'; +import type { MessageSearchParams, SearchHasType } from './useMessageSearch'; +import { useMessageSearch, VALID_HAS_TYPES } from './useMessageSearch'; import { SearchResultGroup } from './SearchResultGroup'; import { SearchInput } from './SearchInput'; import { SearchFilters } from './SearchFilters'; @@ -35,6 +35,7 @@ const useSearchPathSearchParams = (searchParams: URLSearchParams): SearchPathSea order: searchParams.get('order') ?? undefined, rooms: searchParams.get('rooms') ?? undefined, senders: searchParams.get('senders') ?? undefined, + has: searchParams.get('has') ?? undefined, }), [searchParams] ); @@ -55,7 +56,7 @@ export function MessageSearch({ }: Readonly) { const mx = useMatrixClient(); const mDirects = useAtomValue(mDirectAtom); - const allRooms = useRooms(mx, allRoomsAtom, mDirects); + const allRooms = useAtomValue(allRoomsAtom); const [mediaAutoLoad] = useSetting(settingsAtom, 'mediaAutoLoad'); const [urlPreview] = useSetting(settingsAtom, 'urlPreview'); const [legacyUsernameColor] = useSetting(settingsAtom, 'legacyUsernameColor'); @@ -84,6 +85,13 @@ export function MessageSearch({ } return undefined; }, [searchPathSearchParams.senders]); + const searchParamHasTypes = useMemo(() => { + if (!searchPathSearchParams.has) return undefined; + const decoded = decodeSearchParamValueArray(searchPathSearchParams.has).filter( + (t): t is SearchHasType => VALID_HAS_TYPES.includes(t as SearchHasType) + ); + return decoded.length > 0 ? decoded : undefined; + }, [searchPathSearchParams.has]); const msgSearchParams: MessageSearchParams = useMemo(() => { const isGlobal = searchPathSearchParams.global === 'true'; @@ -94,19 +102,31 @@ export function MessageSearch({ order: searchPathSearchParams.order ?? SearchOrderBy.Recent, rooms: searchParamRooms ?? defaultRooms, senders: searchParamsSenders ?? senders, + hasTypes: searchParamHasTypes, }; - }, [searchPathSearchParams, searchParamRooms, searchParamsSenders, rooms, senders]); + }, [ + searchPathSearchParams, + searchParamRooms, + searchParamsSenders, + searchParamHasTypes, + rooms, + senders, + ]); + + const isSearching = + !!msgSearchParams.term || (!!msgSearchParams.hasTypes && msgSearchParams.hasTypes.length > 0); const searchMessages = useMessageSearch(msgSearchParams); const { status, data, error, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteQuery({ - enabled: !!msgSearchParams.term, + enabled: isSearching, queryKey: [ 'search', msgSearchParams.term, msgSearchParams.order, msgSearchParams.rooms, msgSearchParams.senders, + msgSearchParams.hasTypes, ], queryFn: ({ pageParam }) => searchMessages(pageParam), initialPageParam: '', @@ -178,6 +198,28 @@ export function MessageSearch({ }); }; + const handleHasTypesChange = (hasTypes?: SearchHasType[]) => { + setSearchParams((prevParams) => { + const newParams = new URLSearchParams(prevParams); + newParams.delete('has'); + if (hasTypes && hasTypes.length > 0) { + newParams.append('has', encodeSearchParamValueArray(hasTypes)); + } + return newParams; + }); + }; + + const handleSendersChange = (newSenders?: string[]) => { + setSearchParams((prevParams) => { + const newParams = new URLSearchParams(prevParams); + newParams.delete('senders'); + if (newSenders && newSenders.length > 0) { + newParams.append('senders', encodeSearchParamValueArray(newSenders)); + } + return newParams; + }); + }; + const lastVItem = vItems.at(-1); const lastVItemIndex: number | undefined = lastVItem?.index; const lastGroupIndex = groups.length - 1; @@ -224,6 +266,10 @@ export function MessageSearch({ onGlobalChange={handleGlobalChange} order={msgSearchParams.order} onOrderChange={handleOrderChange} + hasTypes={searchParamHasTypes} + onHasTypesChange={handleHasTypesChange} + senders={searchParamsSenders ?? senders} + onSendersChange={handleSendersChange} /> diff --git a/src/app/features/message-search/SearchFilters.tsx b/src/app/features/message-search/SearchFilters.tsx index cc6487e08..bdc649201 100644 --- a/src/app/features/message-search/SearchFilters.tsx +++ b/src/app/features/message-search/SearchFilters.tsx @@ -1,10 +1,13 @@ import type { ChangeEventHandler, MouseEventHandler } from 'react'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import type { RectCords } from 'folds'; import { + Avatar, Box, Chip, Text, + Icon, + Icons, Line, config, PopOut, @@ -17,12 +20,16 @@ import { Input, Badge, } from 'folds'; -import FocusTrap from 'focus-trap-react'; -import { getRoomIconComponent } from '$components/icons/roomIcons'; -import { Check, sizedIcon, PlusCircle, SortAscending, X } from '$components/icons/phosphor'; +import type { IconSrc } from 'folds'; import { SearchOrderBy } from '$types/matrix-sdk'; +import type { RoomMember } from '$types/matrix-sdk'; +import FocusTrap from 'focus-trap-react'; import { useVirtualizer } from '@tanstack/react-virtual'; import { useMatrixClient } from '$hooks/useMatrixClient'; +import { useAtomValue } from 'jotai'; +import { settingsAtom } from '$state/settings'; +import { useClientConfig } from '$hooks/useClientConfig'; +import { getRoomAvatarUrl, getRoomIconSrc } from '$utils/room'; import { factoryRoomIdByAtoZ } from '$utils/sort'; import type { SearchItemStrGetter, UseAsyncSearchOptions } from '$hooks/useAsyncSearch'; import { useAsyncSearch } from '$hooks/useAsyncSearch'; @@ -30,6 +37,9 @@ import type { DebounceOptions } from '$hooks/useDebounce'; import { useDebounce } from '$hooks/useDebounce'; import { VirtualTile } from '$components/virtualizer'; import { stopPropagation } from '$utils/keyboard'; +import { UserAvatar } from '$components/user-avatar'; +import { useMediaAuthentication } from '$hooks/useMediaAuthentication'; +import type { SearchHasType } from './useMessageSearch'; type OrderButtonProps = { order?: string; @@ -93,7 +103,7 @@ function OrderButton({ order, onChange }: OrderButtonProps) { } onClick={handleOpenMenu} > {rankOrder ? Relevance : Recent} @@ -262,10 +272,19 @@ function SelectRoomButton({ roomList, selectedRooms, onChange }: SelectRoomButto size="300" radii="300" aria-pressed={selected} - before={sizedIcon( - getRoomIconComponent(room.getType(), room.getJoinRule()), - '50' - )} + before={ + + } + after={ + mx.isRoomEncrypted(roomId) ? ( + + + + ) : null + } > {room.name} @@ -306,7 +325,7 @@ function SelectRoomButton({ roomList, selectedRooms, onChange }: SelectRoomButto onClick={handleOpenMenu} variant="SurfaceVariant" radii="Pill" - before={sizedIcon(PlusCircle, '100')} + before={} > Select Rooms @@ -314,6 +333,243 @@ function SelectRoomButton({ roomList, selectedRooms, onChange }: SelectRoomButto ); } +const HAS_FILTER_OPTIONS: { type: SearchHasType; label: string; icon: IconSrc }[] = [ + { type: 'image', label: 'Image', icon: Icons.Photo }, + { type: 'file', label: 'File', icon: Icons.File }, + { type: 'audio', label: 'Audio', icon: Icons.VolumeHigh }, + { type: 'video', label: 'Video', icon: Icons.Play }, + { type: 'link', label: 'Link', icon: Icons.Link }, +]; + +type HasFilterChipsProps = { + hasTypes?: SearchHasType[]; + onChange: (hasTypes?: SearchHasType[]) => void; +}; +function HasFilterChips({ hasTypes, onChange }: HasFilterChipsProps) { + const toggle = (type: SearchHasType) => { + if (hasTypes?.includes(type)) { + const next = hasTypes.filter((t) => t !== type); + onChange(next.length > 0 ? next : undefined); + } else { + onChange([...(hasTypes ?? []), type]); + } + }; + + return ( + <> + {HAS_FILTER_OPTIONS.map(({ type, label, icon }) => { + const active = hasTypes?.includes(type); + return ( + : } + outlined + onClick={() => toggle(type)} + > + {label} + + ); + })} + + ); +} + +type SelectSenderButtonProps = { + roomList: string[]; + selectedSenders?: string[]; + onChange: (senders?: string[]) => void; +}; + +const SENDER_SEARCH_OPTS: UseAsyncSearchOptions = { limit: 50, matchOptions: { contain: true } }; +const SENDER_DEBOUNCE_OPTS: DebounceOptions = { wait: 200 }; +const getMemberStr: SearchItemStrGetter = (member, query) => { + const name = member.name ?? member.userId; + return query ? [name, member.userId] : name; +}; +const getMemberDisplayName = (member: RoomMember): string => member.name ?? member.userId; +function SelectSenderButton({ roomList, selectedSenders, onChange }: SelectSenderButtonProps) { + const mx = useMatrixClient(); + const useAuthentication = useMediaAuthentication(); + const [menuAnchor, setMenuAnchor] = useState(); + const scrollRef = useRef(null); + + const members = useMemo(() => { + const seen = new Set(); + const result: RoomMember[] = []; + const scope = roomList.length > 0 ? roomList : []; + for (const roomId of scope) { + const room = mx.getRoom(roomId); + if (!room) continue; + for (const m of room.getMembers()) { + if (!seen.has(m.userId)) { + seen.add(m.userId); + result.push(m); + } + } + } + return result.toSorted((a, b) => + getMemberDisplayName(a).localeCompare(getMemberDisplayName(b)) + ); + }, [mx, roomList]); + + const [searchState, searchMembersRaw, resetSearch] = useAsyncSearch( + members, + getMemberStr, + SENDER_SEARCH_OPTS + ); + const searchMembers = useDebounce(searchMembersRaw, SENDER_DEBOUNCE_OPTS); + const handleSearchChange: ChangeEventHandler = (evt) => { + const value = evt.currentTarget.value.trim(); + if (!value) { + resetSearch(); + return; + } + searchMembers(value); + }; + + const displayMembers = searchState?.items ?? members; + + const virtualizer = useVirtualizer({ + count: displayMembers.length, + getScrollElement: () => scrollRef.current, + estimateSize: () => 32 + 4, + overscan: 10, + }); + const vItems = virtualizer.getVirtualItems(); + + const handleOpenMenu: MouseEventHandler = (evt) => { + setMenuAnchor(evt.currentTarget.getBoundingClientRect()); + }; + + const handleMemberClick: MouseEventHandler = (evt) => { + const userId = evt.currentTarget.getAttribute('data-user-id'); + if (!userId) return; + if (selectedSenders?.includes(userId)) { + const next = selectedSenders.filter((s) => s !== userId); + onChange(next.length > 0 ? next : undefined); + } else { + onChange([...(selectedSenders ?? []), userId]); + } + }; + + return ( + setMenuAnchor(undefined), + clickOutsideDeactivates: true, + escapeDeactivates: stopPropagation, + }} + > + + + + From + 0 ? ( + + {searchState.items.length} + + ) : null + } + /> + + + + {displayMembers.length === 0 && ( + + No members found + + )} +
+ {vItems.map((vItem) => { + const member = displayMembers[vItem.index]!; + const selected = selectedSenders?.includes(member.userId); + return ( + + + } + /> + + } + > + + {getMemberDisplayName(member)} + + + + ); + })} +
+ + + + + + } + > + } + > + Add Sender + + + ); +} + type SearchFiltersProps = { defaultRoomsFilterName: string; allowGlobal?: boolean; @@ -324,6 +580,10 @@ type SearchFiltersProps = { onGlobalChange: (global?: boolean) => void; order?: string; onOrderChange: (order?: string) => void; + hasTypes?: SearchHasType[]; + onHasTypesChange: (hasTypes?: SearchHasType[]) => void; + senders?: string[]; + onSendersChange: (senders?: string[]) => void; }; export function SearchFilters({ defaultRoomsFilterName, @@ -335,7 +595,12 @@ export function SearchFilters({ order, onGlobalChange, onOrderChange, + hasTypes, + onHasTypesChange, + senders, + onSendersChange, }: SearchFiltersProps) { + const senderScope = selectedRooms && selectedRooms.length > 0 ? selectedRooms : roomList; const mx = useMatrixClient(); return ( @@ -345,7 +610,7 @@ export function SearchFilters({ } outlined onClick={() => onGlobalChange()} > @@ -355,7 +620,7 @@ export function SearchFilters({ } outlined onClick={() => onGlobalChange(true)} > @@ -378,8 +643,10 @@ export function SearchFilters({ variant="Success" onClick={() => onSelectedRoomsChange(selectedRooms.filter((rId) => rId !== roomId))} radii="Pill" - before={sizedIcon(getRoomIconComponent(room.getType(), room.getJoinRule()), '50')} - after={sizedIcon(X, '50')} + before={ + + } + after={} > {room.name} @@ -393,6 +660,41 @@ export function SearchFilters({ + + + Has: + + + + + From: + + {senders?.map((sender) => ( + { + const next = senders.filter((s) => s !== sender); + onSendersChange(next.length > 0 ? next : undefined); + }} + radii="Pill" + before={} + after={} + > + {mx.getUser(sender)?.displayName ?? sender} + + ))} + + ); } diff --git a/src/app/features/message-search/searchEncryptedRooms.ts b/src/app/features/message-search/searchEncryptedRooms.ts new file mode 100644 index 000000000..43955dd2b --- /dev/null +++ b/src/app/features/message-search/searchEncryptedRooms.ts @@ -0,0 +1,144 @@ +import { EventType } from '$types/matrix-sdk'; +import type { + IEventWithRoomId, + IResultContext, + MatrixClient, + MatrixEvent, +} from '$types/matrix-sdk'; +import { + HAS_TYPE_TO_MSGTYPE, + SearchHasType, + type ResultGroup, + type ResultItem, +} from './useMessageSearch'; + +/** + * Searches a single room's live timeline for message events that contain + * `term` in their body. Returns a ResultGroup or undefined if no matches. + */ +export function searchRoomTimeline( + room: { roomId: string; getLiveTimeline: () => { getEvents: () => MatrixEvent[] } }, + term: string, + senders?: string[], + hasTypes?: SearchHasType[] +): ResultGroup | undefined { + function toSearchEvent(mEvent: MatrixEvent, roomId: string): IEventWithRoomId { + return { + event_id: mEvent.getId() ?? '', + room_id: roomId, + sender: mEvent.getSender() ?? '', + origin_server_ts: mEvent.getTs(), + content: mEvent.getContent(), + type: mEvent.getType(), + unsigned: mEvent.getUnsigned(), + } as IEventWithRoomId; + } + + function mEventMatchesHasTypes(mEvent: MatrixEvent, hasTypes: SearchHasType[]): boolean { + const content = mEvent.getContent() as { msgtype?: string; body?: string }; + for (const type of hasTypes) { + const msgtype = HAS_TYPE_TO_MSGTYPE[type]; + if (msgtype && content.msgtype === msgtype) return true; + if (type === 'link' && /https?:\/\//i.test(content.body ?? '')) return true; + } + return false; + } + + const events = room.getLiveTimeline().getEvents(); + const items: ResultItem[] = []; + + for (const event of events) { + if (event.getType() !== EventType.RoomMessage) continue; + if (event.isBeingDecrypted() || event.isDecryptionFailure()) continue; + + const sender = event.getSender(); + if (!sender) continue; + if (senders && !senders.includes(sender)) continue; + + if (hasTypes && hasTypes.length > 0 && !mEventMatchesHasTypes(event, hasTypes)) continue; + + if (!event.getId()) continue; + + if (term !== '') { + const body: string = event.getContent().body ?? ''; + if (!body || !body.toLowerCase().includes(term.toLowerCase())) continue; // TODO: fuzzy search? + } + + items.push({ + rank: 1, + event: toSearchEvent(event, room.roomId), + context: { + events_before: [], + events_after: [], + profile_info: {}, + }, + }); + } + + if (items.length === 0) return undefined; + + items.sort((a, b) => b.event.origin_server_ts - a.event.origin_server_ts); + + return { roomId: room.roomId, items }; +} + +/** + * Searches the in-memory live timeline of each listed encrypted room. + * Returns one ResultGroup per room that has at least one match. + */ +export function searchEncryptedRoomsInMemory( + mx: Pick, + term: string, + encryptedRoomIds: string[], + senders?: string[], + hasTypes?: SearchHasType[] +): ResultGroup[] { + const groups: ResultGroup[] = []; + + for (const roomId of encryptedRoomIds) { + const room = mx.getRoom(roomId); + if (!room) continue; + + const group = searchRoomTimeline(room, term, senders, hasTypes); + if (group) groups.push(group); + } + + return groups; +} + +/** + * Splits the user's room filter into encrypted (in-memory) and plaintext (server) buckets. + * + * - When `rooms` is undefined (global search), the server handles plaintext rooms and + * we additionally scan all joined encrypted rooms in memory. + * - When `rooms` is defined, each room is routed to the appropriate search path. + */ +export function partitionRoomsByEncryption( + mx: Pick, + rooms?: string[] +): { encryptedRoomIds: string[]; serverRooms: string[] | undefined; skipServerSearch: boolean } { + let allRooms = mx.getRooms(); + if (rooms === undefined) { + const encryptedRoomIds = allRooms + .filter((r) => r.hasEncryptionStateEvent()) + .map((r) => r.roomId); + return { encryptedRoomIds, serverRooms: undefined, skipServerSearch: false }; + } + + const encryptedRoomIds: string[] = []; + const serverRooms: string[] = []; + + for (const roomId of rooms) { + if (allRooms.find((r) => r.roomId == roomId)?.hasEncryptionStateEvent()) { + encryptedRoomIds.push(roomId); + } else { + serverRooms.push(roomId); + } + } + + return { + encryptedRoomIds, + serverRooms: serverRooms.length > 0 ? serverRooms : undefined, + skipServerSearch: rooms.length > 0 && serverRooms.length === 0, + }; +} diff --git a/src/app/features/message-search/useMessageSearch.ts b/src/app/features/message-search/useMessageSearch.ts index dfe205f69..bab2c600e 100644 --- a/src/app/features/message-search/useMessageSearch.ts +++ b/src/app/features/message-search/useMessageSearch.ts @@ -4,10 +4,91 @@ import type { ISearchRequestBody, ISearchResponse, ISearchResult, + MatrixClient, + MatrixEvent, SearchOrderBy, } from '$types/matrix-sdk'; import { useCallback } from 'react'; import { useMatrixClient } from '$hooks/useMatrixClient'; +import { searchEncryptedRoomsInMemory, partitionRoomsByEncryption } from './searchEncryptedRooms'; +import { useSearchIndex } from '$hooks/useSearchIndex'; +import { useAtomValue } from 'jotai'; +import { settingsAtom } from '$state/settings'; +import { SearchIndexEvent } from '$plugins/search-indexer/types'; + +export function toSearchEvent(mEvent: MatrixEvent, roomId: string): IEventWithRoomId { + return { + event_id: mEvent.getId() ?? '', + room_id: roomId, + sender: mEvent.getSender() ?? '', + origin_server_ts: mEvent.getTs(), + content: mEvent.getContent(), // decrypted content for e2ee events + type: mEvent.getType(), // decrypted event type (e.g. m.room.message, not m.room.encrypted) + unsigned: mEvent.getUnsigned(), + } as IEventWithRoomId; +} + +function idbEventsToGroups( + mx: MatrixClient, + events: SearchIndexEvent[], + order?: string +): ResultGroup[] { + const byRoom = new Map(); + for (const ev of events) { + const liveEvent = mx.getRoom(ev.roomId)?.findEventById(ev.eventId); + const eventData: IEventWithRoomId = liveEvent + ? toSearchEvent(liveEvent, ev.roomId) + : ({ + event_id: ev.eventId, + room_id: ev.roomId, + sender: ev.sender, + origin_server_ts: ev.ts, + content: { + msgtype: ev.msgtype, + body: ev.body, + ...(ev.url !== undefined && { url: ev.url }), + ...(ev.file !== undefined && { file: ev.file }), + ...(ev.info !== undefined && { info: ev.info }), + ...(ev.filename !== undefined && { filename: ev.filename }), + }, + type: 'm.room.message', + unsigned: {}, + } as IEventWithRoomId); + const item: ResultItem = { + rank: 1, + event: eventData, + context: { + events_before: [], + events_after: [], + profile_info: {}, + }, + }; + const arr = byRoom.get(ev.roomId) ?? []; + arr.push(item); + byRoom.set(ev.roomId, arr); + } + + const groups = Array.from(byRoom.entries()).map(([roomId, items]) => ({ + roomId, + // Sort items newest-first so items[0] is always the most recent — required + // for mergeSearchGroups' timestamp comparisons to be correct. + items: + order !== 'rank' + ? items.toSorted( + (a, b) => (b.event.origin_server_ts ?? 0) - (a.event.origin_server_ts ?? 0) + ) + : items, + })); + + // Sort groups newest-first so single-source fast-paths in mergeSearchGroups + // (which return the array unchanged) still produce correct recent order. + return order !== 'rank' + ? groups.toSorted( + (a, b) => + (b.items[0]?.event.origin_server_ts ?? 0) - (a.items[0]?.event.origin_server_ts ?? 0) + ) + : groups; +} export type ResultItem = { rank: number; @@ -26,6 +107,16 @@ export type SearchResult = { groups: ResultGroup[]; }; +export type SearchHasType = 'image' | 'file' | 'audio' | 'video' | 'link'; + +export const VALID_HAS_TYPES: SearchHasType[] = ['image', 'file', 'audio', 'video', 'link']; +export const HAS_TYPE_TO_MSGTYPE: Record = { + image: 'm.image', + file: 'm.file', + audio: 'm.audio', + video: 'm.video', +}; + const groupSearchResult = (results: ISearchResult[]): ResultGroup[] => { const groups: ResultGroup[] = []; @@ -68,19 +159,110 @@ export type MessageSearchParams = { order?: string; rooms?: string[]; senders?: string[]; + hasTypes?: SearchHasType[]; }; export const useMessageSearch = (params: MessageSearchParams) => { const mx = useMatrixClient(); - const { term, order, rooms, senders } = params; + const searchIndex = useSearchIndex(); + const settings = useAtomValue(settingsAtom); + + const { term, order, rooms, senders, hasTypes } = params; + + const filterGroupsByHasType = useCallback( + (grps: ResultGroup[]): ResultGroup[] => { + if (!hasTypes || hasTypes.length === 0) return grps; + const withMsgtype = hasTypes.filter((t) => t !== 'link'); + return grps + .map((g) => ({ + ...g, + items: g.items.filter((item) => { + const content = item.event.content; + if ( + withMsgtype.length > 0 && + withMsgtype.some((t) => content.msgtype === HAS_TYPE_TO_MSGTYPE[t]) + ) + return true; + + if (hasTypes.includes('link') && /https?:\/\//i.test(content.body ?? '')) return true; // TODO: maybe regex isn't the best idea + return false; + }), + })) + .filter((g) => g.items.length > 0); + }, + [hasTypes] + ); + + const mergeSearchGroups = ( + serverGroups: ResultGroup[], + inMemoryGroups: ResultGroup[], + order?: string + ): ResultGroup[] => { + if (inMemoryGroups.length === 0) return serverGroups; + if (serverGroups.length === 0) return inMemoryGroups; + + const all = [...serverGroups, ...inMemoryGroups]; + + if (order === 'rank') { + return all; + } + + return all.toSorted((a, b) => { + const aTs = a.items[0]?.event.origin_server_ts ?? 0; + const bTs = b.items[0]?.event.origin_server_ts ?? 0; + return bTs - aTs; + }); + }; const searchMessages = useCallback( async (nextBatch?: string) => { - if (!term) + const idbSearchAvailable = + settings.idbSearchIndex && !!searchIndex?.ready; + const hasHasTypes = hasTypes && hasTypes.length > 0; + if (!(term || (idbSearchAvailable && hasHasTypes))) return { highlights: [], groups: [], }; - const limit = 20; + + const isFirstPage = !nextBatch || nextBatch === ''; + + if (idbSearchAvailable) { + const idbEvents = await searchIndex.query(term ?? '', { + roomIds: rooms ?? [], + senders, + hasTypes: hasHasTypes ? hasTypes : undefined, + }); + + let foundGroups = idbEventsToGroups(mx, idbEvents, order); + return { + highlights: [], + groups: foundGroups, + }; + } + + const { encryptedRoomIds, serverRooms } = partitionRoomsByEncryption(mx, rooms); + let skipServerSearch = !!!serverRooms; + + let foundGroups: ResultGroup[] = []; + + if (isFirstPage && (rooms ?? []).length > 0) { + if (term || hasHasTypes) { + foundGroups = searchEncryptedRoomsInMemory( + mx, + term ?? '', + encryptedRoomIds, + senders, + hasTypes + ); + } + } + + if (skipServerSearch) { + return { + groups: foundGroups, + highlights: [], + }; + } const requestBody: ISearchRequestBody = { search_categories: { @@ -91,13 +273,12 @@ export const useMessageSearch = (params: MessageSearchParams) => { include_profile: false, }, filter: { - limit, - rooms, senders, + rooms: serverRooms, }, include_state: false, order_by: order as SearchOrderBy.Recent, - search_term: term, + search_term: term ?? '', }, }, }; @@ -106,9 +287,15 @@ export const useMessageSearch = (params: MessageSearchParams) => { body: requestBody, next_batch: nextBatch === '' ? undefined : nextBatch, }); - return parseSearchResult(r); + + const serverResult = parseSearchResult(r); + const filteredServerResult = { + ...serverResult, + groups: mergeSearchGroups(foundGroups, serverResult.groups), + }; + return filteredServerResult; }, - [mx, term, order, rooms, senders] + [mx, term, order, rooms, senders, hasTypes, filterGroupsByHasType] ); return searchMessages; diff --git a/src/app/features/room/RoomViewHeader.tsx b/src/app/features/room/RoomViewHeader.tsx index 632331b17..97ccee897 100644 --- a/src/app/features/room/RoomViewHeader.tsx +++ b/src/app/features/room/RoomViewHeader.tsx @@ -62,7 +62,12 @@ import { useIsDirectRoom, useRoom } from '$hooks/useRoom'; import { useSetting } from '$state/hooks/settings'; import { settingsAtom } from '$state/settings'; import { useSpaceOptionally } from '$hooks/useSpace'; -import { getHomeSearchPath, getSpaceSearchPath, withSearchParam } from '$pages/pathUtils'; +import { + getDirectSearchPath, + getHomeSearchPath, + getSpaceSearchPath, + withSearchParam, +} from '$pages/pathUtils'; import { createLogger } from '$utils/debug'; import { getCanonicalAliasOrRoomId, @@ -562,7 +567,9 @@ export function RoomViewHeader({ callView }: Readonly<{ callView?: boolean }>) { }; const path = space ? getSpaceSearchPath(getCanonicalAliasOrRoomId(mx, space.roomId)) - : getHomeSearchPath(); + : direct + ? getDirectSearchPath() + : getHomeSearchPath(); navigate(withSearchParam(path, searchParams)); }; @@ -681,23 +688,22 @@ export function RoomViewHeader({ callView }: Readonly<{ callView?: boolean }>) { {(!room.isCallRoom() || chat) && ( <> - {!encryptedRoom && ( - - Search - - } - > - {(triggerRef) => ( - - {composerIcon(MagnifyingGlass)} - - )} - - )} + + Search + + } + > + {(triggerRef) => ( + + {composerIcon(MagnifyingGlass)} + + )} + + Type # for rooms, @ for DMs and * for spaces. Hotkey:{' '} {isMacOS() ? KeySymbol.Command : 'Ctrl'} + k - {' / '} - {isMacOS() ? KeySymbol.Command : 'Ctrl'} + f )} diff --git a/src/app/features/settings/developer-tools/DevelopTools.tsx b/src/app/features/settings/developer-tools/DevelopTools.tsx index 100119726..ef71a18a8 100644 --- a/src/app/features/settings/developer-tools/DevelopTools.tsx +++ b/src/app/features/settings/developer-tools/DevelopTools.tsx @@ -15,6 +15,7 @@ import { AccountData } from './AccountData'; import { SyncDiagnostics } from './SyncDiagnostics'; import { DebugLogViewer } from './DebugLogViewer'; import { SentrySettings } from './SentrySettings'; +import { SearchIndexCache } from './SearchIndexCache'; type DeveloperToolsProps = { requestBack?: () => void; @@ -127,6 +128,11 @@ export function DeveloperTools({ requestBack, requestClose }: DeveloperToolsProp )} + {developerTools && ( + + + + )} diff --git a/src/app/features/settings/developer-tools/SearchIndexCache.tsx b/src/app/features/settings/developer-tools/SearchIndexCache.tsx new file mode 100644 index 000000000..af83d1331 --- /dev/null +++ b/src/app/features/settings/developer-tools/SearchIndexCache.tsx @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Box, Button, Text } from 'folds'; +import { SequenceCard } from '$components/sequence-card'; +import { SettingTile } from '$components/setting-tile'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { useSearchIndex } from '$hooks/useSearchIndex'; +import type { SearchIndexState } from '$hooks/useSearchIndex'; +import { SequenceCardStyle } from '$features/settings/styles.css'; + +const LIMIT_OPTIONS: Array<{ label: string; value: number }> = [ + { label: '500 messages', value: 500 }, + { label: '1,000 messages', value: 1000 }, + { label: '2,000 messages (default)', value: 2000 }, + { label: '5,000 messages', value: 5000 }, + { label: 'Unlimited', value: Number.MAX_SAFE_INTEGER }, +]; + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +export function SearchIndexCache() { + const [idbSearchIndex] = useSetting(settingsAtom, 'idbSearchIndex'); + // const [searchIndexMessageLimit, setSearchIndexMessageLimit] = useSetting( + // settingsAtom, + // 'searchIndexMessageLimit' + // ); + const searchIndex = useSearchIndex(); + + const [stats, setStats] = useState(null); + const [clearing, setClearing] = useState(false); + + const refreshStats = useCallback(async () => { + if (!searchIndex?.ready) return; + const s = await searchIndex.state(); + setStats(s); + }, [searchIndex]); + + useEffect(() => { + void refreshStats(); + const id = window.setInterval(() => void refreshStats(), 5000); + return () => window.clearInterval(id); + }, [refreshStats]); + + const handleClear = useCallback(async () => { + if (!searchIndex) return; + setClearing(true); + await searchIndex.clearIndex(); + setStats(null); + setClearing(false); + }, [searchIndex]); + + if (!idbSearchIndex) return null; + + return ( + + Message Search Index + + + {searchIndex?.isBackfilling && ( + + )} + {/* setSearchIndexMessageLimit(Number(e.target.value))} + style={{ padding: '4px 8px', borderRadius: '4px' }} + > + {LIMIT_OPTIONS.map((opt) => ( + + ))} + + } + /> */} + void handleClear()} + variant="Secondary" + fill="Soft" + size="300" + radii="300" + outlined + disabled={clearing} + > + {clearing ? 'Clearing…' : 'Clear'} + + } + /> + + + ); +} diff --git a/src/app/features/settings/experimental/Experimental.tsx b/src/app/features/settings/experimental/Experimental.tsx index fe4b039c7..0b11eb572 100644 --- a/src/app/features/settings/experimental/Experimental.tsx +++ b/src/app/features/settings/experimental/Experimental.tsx @@ -11,6 +11,7 @@ import { Sync } from '../general'; import { SettingsSectionPage } from '../SettingsSectionPage'; import { BandwidthSavingEmojis } from './BandwithSavingEmojis'; import { MSC4268HistoryShare } from './MSC4268HistoryShare'; +import { SearchIDBToggle } from './SearchIDB'; function PersonaToggle() { const [showPersonaSetting, setShowPersonaSetting] = useSetting( @@ -63,6 +64,7 @@ export function Experimental({ requestBack, requestClose }: Readonly + diff --git a/src/app/features/settings/experimental/SearchIDB.tsx b/src/app/features/settings/experimental/SearchIDB.tsx new file mode 100644 index 000000000..bdca9608b --- /dev/null +++ b/src/app/features/settings/experimental/SearchIDB.tsx @@ -0,0 +1,42 @@ +import { SequenceCard } from '$components/sequence-card'; +import { SettingTile } from '$components/setting-tile'; +import { SequenceCardStyle } from '$features/common-settings/styles.css'; +import { useSetting } from '$state/hooks/settings'; +import { settingsAtom } from '$state/settings'; +import { Box, Switch, Text } from 'folds'; + +export function SearchIDBToggle() { + const [enabledIdbSearchIndex, setEnabledIdbSearchIndex] = useSetting( + settingsAtom, + 'idbSearchIndex' + ); + return ( + + Enable Local Message Indexing + + + } + /> + + + ); +} diff --git a/src/app/features/settings/keyboard-shortcuts/KeyboardShortcuts.tsx b/src/app/features/settings/keyboard-shortcuts/KeyboardShortcuts.tsx index ff1ea36ca..7fa0e997e 100644 --- a/src/app/features/settings/keyboard-shortcuts/KeyboardShortcuts.tsx +++ b/src/app/features/settings/keyboard-shortcuts/KeyboardShortcuts.tsx @@ -30,7 +30,10 @@ function formatKey(key: string): string { const SHORTCUT_CATEGORIES: ShortcutCategory[] = [ { name: 'General', - shortcuts: [{ keys: 'Ctrl+F / ⌘+F', description: 'Search for messages' }], + shortcuts: [ + { keys: 'Ctrl+F / ⌘+F', description: 'Search for messages in the current room' }, + { keys: 'Ctrl+Shift+F / ⌘+Shift+F', description: 'Search for messages globally' }, + ], }, { name: 'Navigation', diff --git a/src/app/features/settings/settingsLink.ts b/src/app/features/settings/settingsLink.ts index e3621f413..c641fa26e 100644 --- a/src/app/features/settings/settingsLink.ts +++ b/src/app/features/settings/settingsLink.ts @@ -200,7 +200,12 @@ const settingsLinkFocusIdsBySection: Record { const directMatch = useMatch({ @@ -20,3 +20,13 @@ export const useDirectCreateSelected = (): boolean => { return !!match; }; + +export const useDirectSearchSelected = (): boolean => { + const match = useMatch({ + path: getDirectSearchPath(), + caseSensitive: true, + end: false, + }); + + return !!match; +}; diff --git a/src/app/hooks/useSearchIndex.ts b/src/app/hooks/useSearchIndex.ts new file mode 100644 index 000000000..36b5ff292 --- /dev/null +++ b/src/app/hooks/useSearchIndex.ts @@ -0,0 +1,27 @@ +import { SearchIndexEvent, WorkerMessageTypeIn } from '$plugins/search-indexer/types'; +import { EventType, MatrixEvent, MatrixEventEvent, MsgType } from 'matrix-js-sdk'; +import { createContext, ReactNode, useCallback, useContext, useRef, useState } from 'react'; +import { useMatrixClient } from './useMatrixClient'; + +export type SearchIndexState = { + indexedEventsCount: number; + roomCount: number; + backfillingRoomCount: number; +}; + +export type SearchIndexContextType = { + clearIndex(): unknown; + query: ( + term: string, + opts?: { roomIds?: string[]; senders?: string[]; hasTypes?: string[] } + ) => Promise; + state: () => Promise; + isBackfilling: boolean; + ready: boolean; +}; + +export const SearchIndexContext = createContext(null); + +export function useSearchIndex(): SearchIndexContextType | null { + return useContext(SearchIndexContext); +} diff --git a/src/app/pages/Router.tsx b/src/app/pages/Router.tsx index 56c24a899..db5e5ca78 100644 --- a/src/app/pages/Router.tsx +++ b/src/app/pages/Router.tsx @@ -66,7 +66,7 @@ import { import { ClientBindAtoms, ClientLayout, ClientRoot, ClientRouteOutlet } from './client'; import { HandleNotificationClick, ClientNonUIFeatures } from './client/ClientNonUIFeatures'; import { Home, HomeRouteRoomProvider, HomeSearch } from './client/home'; -import { Direct, DirectCreate, DirectRouteRoomProvider } from './client/direct'; +import { Direct, DirectCreate, DirectRouteRoomProvider, DirectSearch } from './client/direct'; import { RouteSpaceProvider, Space, SpaceRouteRoomProvider, SpaceSearch } from './client/space'; import { Explore, FeaturedRooms, PublicRooms } from './client/explore'; import { Notifications, Inbox, Invites } from './client/inbox'; @@ -269,6 +269,7 @@ export const createRouter = (clientConfig: ClientConfig, screenSize: ScreenSize) > {mobile ? null : } />} } /> + } /> + @@ -889,6 +890,6 @@ export function ClientNonUIFeatures({ children }: ClientNonUIFeaturesProps) { {children} - + ); } diff --git a/src/app/pages/client/direct/Direct.tsx b/src/app/pages/client/direct/Direct.tsx index c9a3bffdf..2b8fd4d86 100644 --- a/src/app/pages/client/direct/Direct.tsx +++ b/src/app/pages/client/direct/Direct.tsx @@ -24,6 +24,7 @@ import { getPhosphorSize, Plus, User, + MagnifyingGlass, } from '$components/icons/phosphor'; import { useVirtualizer } from '@tanstack/react-virtual'; import FocusTrap from 'focus-trap-react'; @@ -40,7 +41,7 @@ import { NavItem, NavItemContent, } from '$components/nav'; -import { getDirectCreatePath, getDirectRoomPath } from '$pages/pathUtils'; +import { getDirectCreatePath, getDirectRoomPath, getDirectSearchPath } from '$pages/pathUtils'; import { getCanonicalAliasOrRoomId } from '$utils/matrix'; import { useSelectedRoom } from '$hooks/router/useSelectedRoom'; import { VirtualTile } from '$components/virtualizer'; @@ -60,7 +61,7 @@ import { getRoomNotificationMode, useRoomsNotificationPreferencesContext, } from '$hooks/useRoomsNotificationPreferences'; -import { useDirectCreateSelected } from '$hooks/router/useDirectSelected'; +import { useDirectCreateSelected, useDirectSearchSelected } from '$hooks/router/useDirectSelected'; import { useDirectRooms } from './useDirectRooms'; import { SidebarResizer } from '$pages/client/sidebar/SidebarResizer'; import { useScreenSizeContext, ScreenSize } from '$hooks/useScreenSize'; @@ -211,6 +212,7 @@ export function Direct() { const [joinCallOnSingleClick] = useSetting(settingsAtom, 'joinCallOnSingleClick'); const createDirectSelected = useDirectCreateSelected(); + const directSearchSelected = useDirectSearchSelected(); const selectedRoomId = useSelectedRoom(); const noRoomToDisplay = directs.length === 0; @@ -311,6 +313,30 @@ export function Direct() { + + navigate(getDirectSearchPath())}> + + + + {menuIcon(MagnifyingGlass)} + + {!hideText && ( + + + Message Search + + + )} + + + + diff --git a/src/app/pages/client/direct/Search.tsx b/src/app/pages/client/direct/Search.tsx new file mode 100644 index 000000000..bc6eb4620 --- /dev/null +++ b/src/app/pages/client/direct/Search.tsx @@ -0,0 +1,54 @@ +import { useRef } from 'react'; +import { Box, Icon, Icons, Text, Scroll, IconButton } from 'folds'; +import { Page, PageContent, PageContentCenter, PageHeader } from '$components/page'; +import { MessageSearch } from '$features/message-search'; +import { ScreenSize, useScreenSizeContext } from '$hooks/useScreenSize'; +import { BackRouteHandler } from '$components/BackRouteHandler'; +import { useDirectRooms } from './useDirectRooms'; + +export function DirectSearch() { + const scrollRef = useRef(null); + const rooms = useDirectRooms(); + const screenSize = useScreenSizeContext(); + + return ( + + + + + {screenSize === ScreenSize.Mobile && ( + + {(onBack) => ( + + + + )} + + )} + + + {screenSize !== ScreenSize.Mobile && } + + Message Search + + + + + + + + + + + + + + + + ); +} diff --git a/src/app/pages/client/direct/index.ts b/src/app/pages/client/direct/index.ts index d247bbc03..770aa28f2 100644 --- a/src/app/pages/client/direct/index.ts +++ b/src/app/pages/client/direct/index.ts @@ -1,3 +1,4 @@ export * from './Direct'; export * from './RoomProvider'; export * from './DirectCreate'; +export * from './Search'; diff --git a/src/app/pages/pathUtils.ts b/src/app/pages/pathUtils.ts index 4a95f47fc..9401fb9ed 100644 --- a/src/app/pages/pathUtils.ts +++ b/src/app/pages/pathUtils.ts @@ -28,6 +28,7 @@ import { SPACE_ROOM_PATH, SPACE_SEARCH_PATH, CREATE_PATH, + DIRECT_SEARCH_PATH, } from './paths'; export const joinPathComponent = (path: Path): string => path.pathname + path.search + path.hash; @@ -102,6 +103,7 @@ export const getHomeRoomPath = (roomIdOrAlias: string, eventId?: string): string export const getDirectPath = (): string => DIRECT_PATH; export const getDirectCreatePath = (): string => DIRECT_CREATE_PATH; +export const getDirectSearchPath = (): string => DIRECT_SEARCH_PATH; export const getDirectRoomPath = (roomIdOrAlias: string, eventId?: string): string => { const params = { roomIdOrAlias: encodeURIComponent(roomIdOrAlias), diff --git a/src/app/pages/paths.ts b/src/app/pages/paths.ts index 1ac57b756..aa0a3feeb 100644 --- a/src/app/pages/paths.ts +++ b/src/app/pages/paths.ts @@ -37,6 +37,7 @@ export type SearchPathSearchParams = { order?: string; rooms?: string; senders?: string; + has?: string; }; export const SEARCH_PATH_SEGMENT = 'search/'; @@ -58,6 +59,7 @@ export type DirectCreateSearchParams = { }; export const DIRECT_CREATE_PATH = `/direct/${CREATE_PATH_SEGMENT}`; export const DIRECT_ROOM_PATH = `/direct/${ROOM_PATH_SEGMENT}`; +export const DIRECT_SEARCH_PATH = `/direct/${SEARCH_PATH_SEGMENT}`; export const SPACE_PATH = '/:spaceIdOrAlias/'; export const SPACE_LOBBY_PATH = `/:spaceIdOrAlias/${LOBBY_PATH_SEGMENT}`; diff --git a/src/app/plugins/search-indexer/index.ts b/src/app/plugins/search-indexer/index.ts new file mode 100644 index 000000000..125aabfc0 --- /dev/null +++ b/src/app/plugins/search-indexer/index.ts @@ -0,0 +1,334 @@ +import { Document, IndexedDB, Resolver } from 'flexsearch'; +import { + BackfillState, + IndexWorkerMessageIn, + IndexWorkerMessageOut, + SearchIndexEvent, + WorkerMessageTypeIn, + WorkerMessageTypeOut, +} from './types'; + +export const HAS_TYPE_TO_MSGTYPE: Record = { + image: 'm.image', + file: 'm.file', + audio: 'm.audio', + video: 'm.video', +}; + +function openIdb(name: string): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(name, 1); + + req.onupgradeneeded = (e) => { + const db = req.result; + + db.createObjectStore('index'); + db.createObjectStore('backfill'); + }; + + req.addEventListener('success', () => resolve(req.result)); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbGet(db: IDBDatabase, store: string, key: string): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readonly'); + const req = tx.objectStore(store).get(key); + req.addEventListener('success', () => resolve(req.result)); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbPut(db: IDBDatabase, store: string, key: string, value: unknown): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite'); + const req = tx.objectStore(store).put(value, key); + req.addEventListener('success', () => resolve()); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function idbGetAll(db: IDBDatabase, store: string): Promise<{ key: string; value: unknown }[]> { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readonly'); + const results: { key: string; value: unknown }[] = []; + const keysReq = tx.objectStore(store).openCursor(); + keysReq.addEventListener('success', () => { + const cursor = keysReq.result; + if (cursor) { + results.push({ key: cursor.key as string, value: cursor.value as unknown }); + cursor.continue(); + } else { + resolve(results); + } + }); + keysReq.addEventListener('error', () => reject(keysReq.error)); + }); +} + +function idbClear(db: IDBDatabase, store: string): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(store, 'readwrite'); + const req = tx.objectStore(store).clear(); + req.addEventListener('success', () => resolve()); + req.addEventListener('error', () => reject(req.error)); + }); +} + +function getDocument( + data: Record | undefined = undefined +): Document { + const document = new Document({ + document: { + id: 'eventId', + store: true, + index: ['body'], + }, + }); + + if (data) { + for (const [key, value] of Object.entries(data)) { + document.import(key, value); + } + } + + return document; +} + +let dirty = false; +const FLUSH_DEBOUNCE_MS = 5000; +let flushTimer: ReturnType | null = null; +let flushTries = 0 + +let db: IDBDatabase | null = null; +let document: Document | null = null; + +const roomQueues = new Map>(); + +async function flushIndex(): Promise { + if (!db || !document || !dirty) return; + try { + const doFlush = async () => { + if (!db || !document) return; + document.export(async (key, data) => { + if (db) await idbPut(db, 'index', key, data); + }); + + const roomQueuesData: Record> = {}; + for (const [roomId, queue] of roomQueues.entries()) { + roomQueuesData[roomId] = queue; + } + await idbPut(db, 'index', 'rooms', roomQueuesData); + dirty = false; + }; + if ('locks' in navigator) { + await navigator.locks.request('sable-search-index-writer', { mode: 'exclusive' }, doFlush); + } else { + await doFlush(); + } + } catch {} +} + +function scheduleFlush(): void { + if (flushTimer !== null && flushTries <= 100) { + clearTimeout(flushTimer) + flushTries += 1; + }; + flushTimer = setTimeout(() => { + flushTimer = null; + flushTries = 0; + void flushIndex(); + }, FLUSH_DEBOUNCE_MS); +} + +function post(msg: IndexWorkerMessageOut): void { + self.postMessage(msg); +} + +function makeTypeFilter( + hasTypes: string[] | undefined +): ((ev: SearchIndexEvent) => boolean) | null { + if (!hasTypes || hasTypes.length === 0) return null; + const allowedMsgtypes = hasTypes.filter((v) => v != 'link').map((t) => HAS_TYPE_TO_MSGTYPE[t]); + const needsLink = hasTypes.includes('link'); + return (ev: SearchIndexEvent) => { + if (allowedMsgtypes.includes(ev.msgtype)) return true; + if (needsLink && ev.hasLink) return true; + return false; + }; +} + +self.addEventListener('message', async (event: MessageEvent) => { + const msg = event.data; + + switch (msg.type) { + case WorkerMessageTypeIn.Init: + const dbName = `sable-search${msg.userId}`; + + db = await openIdb(dbName); + + const serialized = await idbGetAll(db, 'index'); + if (serialized) { + const records: Record = {}; + for (const { key, value } of serialized) { + records[key] = value as string; + } + document = getDocument(records); + const savedQueues = await idbGet>>( + db, + 'index', + 'rooms' + ); + if (savedQueues) { + for (const [roomId, queue] of Object.entries(savedQueues)) { + roomQueues.set(roomId, queue); + for (const [eventId] of queue) { + const fields = document.get(eventId); + } + } + } + } else { + document = getDocument(); + } + + post({ + type: WorkerMessageTypeOut.Ready, + //@ts-expect-error flexsearch types are wrong for some reason + indexedEventCount: document.store.size, + roomCount: roomQueues.size, + }); + break; + + case WorkerMessageTypeIn.Query: + if (!document) { + post({ type: WorkerMessageTypeOut.QueryResult, id: msg.id, events: [] }); + return; + } + + const typeFilter = makeTypeFilter(msg.hasTypes); + + function matchesFilters(ev: SearchIndexEvent): boolean { + if (msg.type != WorkerMessageTypeIn.Query) return false; + if (msg.roomIds && msg.roomIds.length > 0 && !msg.roomIds.includes(ev.roomId)) return false; + if (msg.senders && msg.senders.length > 0 && !msg.senders.includes(ev.sender)) return false; + if (typeFilter && !typeFilter(ev)) return false; + return true; + } + + if (!msg.term) { + const results: SearchIndexEvent[] = + //@ts-expect-error flexsearch types are very bad + [...document.store.values()] + .filter((v) => matchesFilters(v)) + .slice(0, 1000); + post({ type: WorkerMessageTypeOut.QueryResult, id: msg.id, events: results }); + return; + } + + + let result = + document.search(msg.term, { + //@ts-expect-error flexsearch types are very bad + limit: document.store.size, + enrich: true, + }) + .flatMap((r) => r.result.map((v) => v.doc!)) + .filter((r) => r != undefined) + .filter((v) => matchesFilters(v)) + .slice(0, 1000); + + post({ type: WorkerMessageTypeOut.QueryResult, id: msg.id, events: result! }); + + case WorkerMessageTypeIn.State: + if (!document) { + post({ type: WorkerMessageTypeOut.State, indexedEventCount: 0, roomCount: 0 }); + return; + } + post({ + type: WorkerMessageTypeOut.State, + //@ts-expect-error flexsearch types are very bad + indexedEventCount: document.store.size, + roomCount: roomQueues.size, + }); + break; + case WorkerMessageTypeIn.Clear: + if (!db) return; + Promise.all([idbClear(db, 'index'), idbClear(db, 'backfill')]); + break; + case WorkerMessageTypeIn.SetBackfillState: + if (!db) return; + await idbPut(db, 'backfill', msg.roomId, msg.state); + break; + case WorkerMessageTypeIn.GetBackfillStates: + if (!db) { + post({ type: WorkerMessageTypeOut.BackfillStatesDone, states: {} }); + return; + } + + const rows = await idbGetAll(db, 'backfill'); + const states: Record = {}; + for (const { key, value } of rows) { + states[key] = value as BackfillState; + } + post({ type: WorkerMessageTypeOut.BackfillStatesDone, states }); + break; + case WorkerMessageTypeIn.Index: + if (!document) return; + + for (const ev of msg.events) { + if (!ev.eventId || !ev.body.trim()) continue; + if (document.contain(ev.eventId)) continue; + document.add(ev); + + let queue = roomQueues.get(ev.roomId); + if (!queue) { + queue = []; + roomQueues.set(ev.roomId, queue); + } + const lastEntry = queue[queue.length - 1]; + if (queue.length === 0 || (lastEntry !== undefined && lastEntry[1] <= ev.ts)) { + queue.push([ev.eventId, ev.ts]); + } else { + let lo = 0; + let hi = queue.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + const midEntry = queue[mid]; + if (midEntry !== undefined && midEntry[1] <= ev.ts) lo = mid + 1; + else hi = mid; + } + queue.splice(lo, 0, [ev.eventId, ev.ts]); + } + } + dirty = true; + scheduleFlush(); + break; + case WorkerMessageTypeIn.RedactEvents: + if (!document) return; + for (const ev of msg.eventIds) { + document.remove(ev) + } + dirty = true; + scheduleFlush(); + break + case WorkerMessageTypeIn.EditEvents: + if (!document) return; + for (const id in msg.events) { + if (!msg.events[id]) continue + document.set(id, msg.events[id]) + } + dirty = true; + scheduleFlush(); + break + case WorkerMessageTypeIn.Flush: + void flushIndex().then(() => { + post({ + type: WorkerMessageTypeOut.FlushDone, + }); + }); + break; + default: + break; + } +}); diff --git a/src/app/plugins/search-indexer/types.ts b/src/app/plugins/search-indexer/types.ts new file mode 100644 index 000000000..3f814f82b --- /dev/null +++ b/src/app/plugins/search-indexer/types.ts @@ -0,0 +1,112 @@ +import { IEncryptedFile, IFileInfo, IThumbnailContent } from '$types/matrix/common'; + +export type SearchIndexEvent = { + eventId: string; + roomId: string; + sender: string; + + msgtype: string; + body: string; + ts: number; + + hasLink: boolean; + filename?: string; + url?: string; + info?: IFileInfo; + file?: any; +}; + +export enum WorkerMessageTypeIn { + Init, + GetBackfillStates, + SetBackfillState, + EditEvents, + RedactEvents, + Index, + Query, + State, + Clear, + Flush, +} + +export type IndexWorkerMessageIn = + | { + type: WorkerMessageTypeIn.Init; + userId: string; + } + | { + type: WorkerMessageTypeIn.GetBackfillStates; + } + | { + type: WorkerMessageTypeIn.Query; + id: string; + term?: string; + roomIds?: string[]; + senders?: string[]; + hasTypes?: string[]; + } + | { + type: WorkerMessageTypeIn.SetBackfillState; + roomId: string; + state: BackfillState; + } + |{ + type: WorkerMessageTypeIn.RedactEvents; + eventIds: string[] + } + |{ + type: WorkerMessageTypeIn.EditEvents; + events: Record + } + | { + type: WorkerMessageTypeIn.Index; + events: SearchIndexEvent[]; + } + | { + type: WorkerMessageTypeIn.State; + } + | { + type: WorkerMessageTypeIn.Clear; + } + | { + type: WorkerMessageTypeIn.Flush; + }; + +export enum WorkerMessageTypeOut { + Ready, + BackfillStatesDone, + QueryResult, + State, + FlushDone, +} + +export type IndexWorkerMessageOut = + | { + type: WorkerMessageTypeOut.Ready; + indexedEventCount: number; + roomCount: number; + } + | { + type: WorkerMessageTypeOut.BackfillStatesDone; + states: Record; + } + | { + type: WorkerMessageTypeOut.QueryResult; + id: string; + events: SearchIndexEvent[]; + } + | { + type: WorkerMessageTypeOut.State; + indexedEventCount: number; + roomCount: number; + } + | { + type: WorkerMessageTypeOut.FlushDone; + }; + +export type BackfillState = { + token: string | null; + done: boolean; + indexedCount: number; + oldestTs?: number; +}; diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index 0ec9d3bb2..afda52b09 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -141,6 +141,7 @@ export interface Settings { developerTools: boolean; enableMSC4268CMD: boolean; settingsSyncEnabled: boolean; + idbSearchIndex: boolean; // Cosmetics! iconCompactSizePx: number; @@ -296,6 +297,7 @@ export const defaultSettings: Settings = { developerTools: false, settingsSyncEnabled: false, + idbSearchIndex: false, // Cosmetics! iconCompactSizePx: 16, diff --git a/src/app/utils/room.ts b/src/app/utils/room.ts index 6d201f34a..5ecffb067 100644 --- a/src/app/utils/room.ts +++ b/src/app/utils/room.ts @@ -23,11 +23,13 @@ import { MsgType, KnownMembership, RoomType, + JoinRule, } from '$types/matrix-sdk'; import type { IRoomCreateContent, RoomToParents, UnreadInfo } from '$types/matrix/room'; import { NotificationType } from '$types/matrix/room'; import * as Sentry from '@sentry/react'; +import { IconName, IconSrc } from 'folds'; export const getStateEvent = ( room: Room, @@ -510,6 +512,46 @@ export const getUnreadInfos = (mx: MatrixClient, options?: UnreadInfoOptions): U return unreadInfos; }; +export const getRoomIconSrc = ( + icons: Record, + roomType?: string, + joinRule?: JoinRule +): IconSrc => { + if (roomType === RoomType.Space) { + if (joinRule === JoinRule.Public) return icons.SpaceGlobe; + if ( + joinRule === JoinRule.Invite || + joinRule === JoinRule.Knock || + joinRule === JoinRule.Private + ) { + return icons.SpaceLock; + } + return icons.Space; + } + + if (roomType === RoomType.UnstableCall) { + if (joinRule === JoinRule.Public) return icons.VolumeHighGlobe; + if ( + joinRule === JoinRule.Invite || + joinRule === JoinRule.Knock || + joinRule === JoinRule.Private + ) { + return icons.VolumeHighLock; + } + return icons.VolumeHigh; + } + + if (joinRule === JoinRule.Public) return icons.HashGlobe; + if ( + joinRule === JoinRule.Invite || + joinRule === JoinRule.Knock || + joinRule === JoinRule.Private + ) { + return icons.HashLock; + } + return icons.Hash; +}; + export const getRoomAvatarUrl = ( mx: MatrixClient, room: Room,