Skip to content

Apps/plugins cleanup #2416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 64 additions & 62 deletions backend/database/apps.py

Large diffs are not rendered by default.

43 changes: 27 additions & 16 deletions backend/database/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ def add_message(uid: str, message_data: dict):
return message_data


def add_plugin_message(text: str, plugin_id: str, uid: str, conversation_id: Optional[str] = None) -> Message:
def add_app_message(text: str, app_id: str, uid: str, conversation_id: Optional[str] = None) -> Message:
ai_message = Message(
id=str(uuid.uuid4()),
text=text,
created_at=datetime.now(timezone.utc),
sender='ai',
plugin_id=plugin_id,
app_id=app_id,
from_external_integration=False,
type='text',
memories_id=[conversation_id] if conversation_id else [],
Expand All @@ -40,7 +40,7 @@ def add_summary_message(text: str, uid: str) -> Message:
text=text,
created_at=datetime.now(timezone.utc),
sender='ai',
plugin_id=None,
app_id=None,
from_external_integration=False,
type='day_summary',
memories_id=[],
Expand All @@ -49,11 +49,11 @@ def add_summary_message(text: str, uid: str) -> Message:
return ai_message


def get_plugin_messages(uid: str, plugin_id: str, limit: int = 20, offset: int = 0, include_conversations: bool = False):
def get_app_messages(uid: str, app_id: str, limit: int = 20, offset: int = 0, include_conversations: bool = False):
user_ref = db.collection('users').document(uid)
messages_ref = (
user_ref.collection('messages')
.where(filter=FieldFilter('plugin_id', '==', plugin_id))
.where(filter=FieldFilter('plugin_id', '==', app_id))
.order_by('created_at', direction=firestore.Query.DESCENDING)
.limit(limit)
.offset(offset)
Expand Down Expand Up @@ -87,25 +87,27 @@ def get_plugin_messages(uid: str, plugin_id: str, limit: int = 20, offset: int =
# Attach conversations to messages
for message in messages:
message['memories'] = [
conversations[conversation_id] for conversation_id in message.get('memories_id', []) if conversation_id in conversations
conversations[conversation_id] for conversation_id in message.get('memories_id', []) if
conversation_id in conversations
]

return messages


@timeit
def get_messages(
uid: str, limit: int = 20, offset: int = 0, include_conversations: bool = False, plugin_id: Optional[str] = None, chat_session_id: Optional[str] = None
uid: str, limit: int = 20, offset: int = 0, include_conversations: bool = False, app_id: Optional[str] = None,
chat_session_id: Optional[str] = None
# include_plugin_id_filter: bool = True,
):
print('get_messages', uid, limit, offset, plugin_id, include_conversations)
print('get_messages', uid, limit, offset, app_id, include_conversations)
user_ref = db.collection('users').document(uid)
messages_ref = (
user_ref.collection('messages')
.where(filter=FieldFilter('deleted', '==', False))
)
# if include_plugin_id_filter:
messages_ref = messages_ref.where(filter=FieldFilter('plugin_id', '==', plugin_id))
messages_ref = messages_ref.where(filter=FieldFilter('plugin_id', '==', app_id))
if chat_session_id:
messages_ref = messages_ref.where(filter=FieldFilter('chat_session_id', '==', chat_session_id))

Expand Down Expand Up @@ -140,7 +142,8 @@ def get_messages(
# Attach conversations to messages
for message in messages:
message['memories'] = [
conversations[conversation_id] for conversation_id in message.get('memories_id', []) if conversation_id in conversations
conversations[conversation_id] for conversation_id in message.get('memories_id', []) if
conversation_id in conversations
]

# Fetch file chat
Expand Down Expand Up @@ -190,15 +193,16 @@ def report_message(uid: str, msg_doc_id: str):
return {"message": f"Update failed: {e}"}


def batch_delete_messages(parent_doc_ref, batch_size=450, plugin_id: Optional[str] = None, chat_session_id: Optional[str] = None):
def batch_delete_messages(parent_doc_ref, batch_size=450, app_id: Optional[str] = None,
chat_session_id: Optional[str] = None):
messages_ref = (
parent_doc_ref.collection('messages')
.where(filter=FieldFilter('deleted', '==', False))
)
messages_ref = messages_ref.where(filter=FieldFilter('plugin_id', '==', plugin_id))
messages_ref = messages_ref.where(filter=FieldFilter('plugin_id', '==', app_id))
if chat_session_id:
messages_ref = messages_ref.where(filter=FieldFilter('chat_session_id', '==', chat_session_id))
print('batch_delete_messages', plugin_id)
print('batch_delete_messages', app_id)
last_doc = None # For pagination

while True:
Expand Down Expand Up @@ -234,11 +238,12 @@ def clear_chat(uid: str, app_id: Optional[str] = None, chat_session_id: Optional
print(f"Deleting messages for user: {uid}")
if not user_ref.get().exists:
return {"message": "User not found"}
batch_delete_messages(user_ref, plugin_id=app_id, chat_session_id=chat_session_id)
batch_delete_messages(user_ref, app_id=app_id, chat_session_id=chat_session_id)
return None
except Exception as e:
return {"message": str(e)}


def add_multi_files(uid: str, files_data: list):
batch = db.batch()
user_ref = db.collection('users').document(uid)
Expand All @@ -250,6 +255,7 @@ def add_multi_files(uid: str, files_data: list):

batch.commit()


def get_chat_files(uid: str, files_id: List[str] = []):
files_ref = (
db.collection('users').document(uid).collection('files')
Expand All @@ -272,17 +278,19 @@ def delete_multi_files(uid: str, files_data: list):

batch.commit()


def add_chat_session(uid: str, chat_session_data: dict):
chat_session_data['deleted'] = False
user_ref = db.collection('users').document(uid)
user_ref.collection('chat_sessions').document(chat_session_data['id']).set(chat_session_data)
return chat_session_data

def get_chat_session(uid: str, plugin_id: Optional[str] = None):

def get_chat_session(uid: str, app_id: Optional[str] = None):
session_ref = (
db.collection('users').document(uid).collection('chat_sessions')
.where(filter=FieldFilter('deleted', '==', False))
.where(filter=FieldFilter('plugin_id', '==', plugin_id))
.where(filter=FieldFilter('plugin_id', '==', app_id))
.limit(1)
)

Expand All @@ -292,16 +300,19 @@ def get_chat_session(uid: str, plugin_id: Optional[str] = None):

return None


def delete_chat_session(uid, chat_session_id):
user_ref = db.collection('users').document(uid)
session_ref = user_ref.collection('chat_sessions').document(chat_session_id)
session_ref.update({'deleted': True})


def add_message_to_chat_session(uid: str, chat_session_id: str, message_id: str):
user_ref = db.collection('users').document(uid)
session_ref = user_ref.collection('chat_sessions').document(chat_session_id)
session_ref.update({"message_ids": firestore.ArrayUnion([message_id])})


def add_files_to_chat_session(uid: str, chat_session_id: str, file_ids: List[str]):
if not file_ids:
return
Expand Down
10 changes: 5 additions & 5 deletions backend/database/mem_db.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import time

proactive_noti_sent_at = {} # {<uid:plugin_id>: (ts, ex)}
proactive_noti_sent_at = {} # {<uid:app_id>: (ts, ex)}

def set_proactive_noti_sent_at(uid: str, plugin_id: str, ts: int, ttl: int = 30):
k = f'{uid}:{plugin_id}'
def set_proactive_noti_sent_at(uid: str, app_id: str, ts: int, ttl: int = 30):
k = f'{uid}:{app_id}'
proactive_noti_sent_at[k] = (ts, ttl + time.time())

def get_proactive_noti_sent_at(uid: str, plugin_id: str):
k = f'{uid}:{plugin_id}'
def get_proactive_noti_sent_at(uid: str, app_id: str):
k = f'{uid}:{app_id}'
if k not in proactive_noti_sent_at:
return None

Expand Down
52 changes: 26 additions & 26 deletions backend/database/redis_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,17 +185,17 @@ def get_specific_user_review(app_id: str, uid: str) -> dict:
return reviews.get(uid, {})


def migrate_user_plugins_reviews(prev_uid: str, new_uid: str):
def migrate_user_apps_reviews(prev_uid: str, new_uid: str):
for key in r.scan_iter(f'plugins:*:reviews'):
plugin_id = key.decode().split(':')[1]
app_id = key.decode().split(':')[1]
reviews = r.get(key)
if not reviews:
continue
reviews = eval(reviews)
if prev_uid in reviews:
reviews[new_uid] = reviews.pop(prev_uid)
reviews[new_uid]['uid'] = new_uid
r.set(f'plugins:{plugin_id}:reviews', str(reviews))
r.set(f'plugins:{app_id}:reviews', str(reviews))


def set_user_paid_app(app_id: str, uid: str, ttl: int):
Expand All @@ -217,36 +217,36 @@ def disable_app(uid: str, app_id: str):
r.srem(f'users:{uid}:enabled_plugins', app_id)


def get_enabled_plugins(uid: str):
def get_enabled_apps(uid: str):
val = r.smembers(f'users:{uid}:enabled_plugins')
if not val:
return []
return [x.decode() for x in val]


def get_plugin_reviews(plugin_id: str) -> dict:
reviews = r.get(f'plugins:{plugin_id}:reviews')
def get_app_reviews(app_id: str) -> dict:
reviews = r.get(f'plugins:{app_id}:reviews')
if not reviews:
return {}
return eval(reviews)


def get_plugins_reviews(plugin_ids: list) -> dict:
if not plugin_ids:
def get_apps_reviews(app_ids: list) -> dict:
if not app_ids:
return {}

keys = [f'plugins:{plugin_id}:reviews' for plugin_id in plugin_ids]
keys = [f'plugins:{app_id}:reviews' for app_id in app_ids]
reviews = r.mget(keys)
if reviews is None:
return {}
return {
plugin_id: eval(review) if review else {}
for plugin_id, review in zip(plugin_ids, reviews)
app_id: eval(review) if review else {}
for app_id, review in zip(app_ids, reviews)
}


def set_plugin_installs_count(plugin_id: str, count: int):
r.set(f'plugins:{plugin_id}:installs', count)
def set_app_installs_count(app_id: str, count: int):
r.set(f'plugins:{app_id}:installs', count)


def increase_app_installs_count(app_id: str):
Expand All @@ -257,24 +257,24 @@ def decrease_app_installs_count(app_id: str):
r.decr(f'plugins:{app_id}:installs')


def get_plugin_installs_count(plugin_id: str) -> int:
count = r.get(f'plugins:{plugin_id}:installs')
def get_app_installs_count(app_id: str) -> int:
count = r.get(f'plugins:{app_id}:installs')
if not count:
return 0
return int(count)


def get_plugins_installs_count(plugin_ids: list) -> dict:
if not plugin_ids:
def get_apps_installs_count(app_ids: list) -> dict:
if not app_ids:
return {}

keys = [f'plugins:{plugin_id}:installs' for plugin_id in plugin_ids]
keys = [f'plugins:{app_id}:installs' for app_id in app_ids]
counts = r.mget(keys)
if counts is None:
return {}
return {
plugin_id: int(count) if count else 0
for plugin_id, count in zip(plugin_ids, counts)
app_id: int(count) if count else 0
for app_id, count in zip(app_ids, counts)
}


Expand Down Expand Up @@ -460,19 +460,19 @@ def has_migrated_retrieval_conversation_id(conversation_id: str) -> bool:
return r.sismember('migrated_retrieval_memory_ids', conversation_id)


def set_proactive_noti_sent_at(uid: str, plugin_id: str, ts: int, ttl: int = 30):
r.set(f'{uid}:{plugin_id}:proactive_noti_sent_at', ts, ex=ttl)
def set_proactive_noti_sent_at(uid: str, app_id: str, ts: int, ttl: int = 30):
r.set(f'{uid}:{app_id}:proactive_noti_sent_at', ts, ex=ttl)


def get_proactive_noti_sent_at(uid: str, plugin_id: str):
val = r.get(f'{uid}:{plugin_id}:proactive_noti_sent_at')
def get_proactive_noti_sent_at(uid: str, app_id: str):
val = r.get(f'{uid}:{app_id}:proactive_noti_sent_at')
if not val:
return None
return int(val)


def get_proactive_noti_sent_at_ttl(uid: str, plugin_id: str):
return r.ttl(f'{uid}:{plugin_id}:proactive_noti_sent_at')
def get_proactive_noti_sent_at_ttl(uid: str, app_id: str):
return r.ttl(f'{uid}:{app_id}:proactive_noti_sent_at')


def set_user_preferred_app(uid: str, app_id: str):
Expand Down
Loading