uhhh

main.py

@@ -7,6 +7,7 @@ import meilisearch
 import os
 
 from aiobotocore.session import get_session as aiobotocore_get_session
+from db import attachment
 from db.attachment import Attachment
 from db.meme import Meme
 from db.migrate import migrate_db
@@ -94,9 +95,9 @@ async def process_channel(
                 return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
 
     def create_attachments(message, channel, guild):
-        urls = [parse_url(attachment['url']) for attachment in message['attachments']]
+        urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
 
-        def generate_attachment(url: Url):
+        def generate_attachment(url: Url, attachment_id: str):
             if not url.path:
                 return None
 
@@ -107,6 +108,7 @@ async def process_channel(
 
             return Attachment(
                 id=uuid4(),
+                attachment_id=int(attachment_id),
                 msg_id=int(message['id']),
                 message=message,
                 guild=guild,
@@ -114,7 +116,7 @@ async def process_channel(
                 channel=channel
             )
 
-        return [generate_attachment(url) for url in urls]
+        return [generate_attachment(*url) for url in urls]
 
     async def upload_and_persist(attachment: Attachment, session: AsyncSession):
         try:
@@ -155,10 +157,19 @@ async def process_channel(
 
     print(f"found {len(attachments)} attachments for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
 
+    attachment_ids = list(map(lambda a: a.attachment_id, attachments))
+
     async with db_session() as session:
-        results = await session.execute(select(Attachment.msg_id).distinct())
-        msg_ids = [row[0] for row in results.fetchall()]
-        new_attachments = list(filter(lambda a: a.msg_id not in msg_ids, attachments))
+        # this is a bit weird
+        # we find the union of attachment_ids in the db and our list
+        results = await session.execute(
+            select(Attachment.attachment_id)
+                .filter(Attachment.attachment_id.in_(attachment_ids)))
+
+        existing_ids = [row[0] for row in results.fetchall()]
+
+        # and then filter out msgs that are already processed
+        new_attachments = list(filter(lambda a: a.attachment_id not in existing_ids, attachments))
         print(f"{len(new_attachments)} attachments to download for channel guild={guild['name']} name={channel['name']} id={channel['id']}");
 
         # ok now we will try and upload each attachment