From ecd8f78fe4a299e3c99ff90e35f9eac37a2d392c Mon Sep 17 00:00:00 2001
From: ntr <ntr@smokestack.io>
Date: Tue, 23 Apr 2024 00:28:25 +1000
Subject: [PATCH] add content-type to uploads

---
 src/attachments.ts | 11 +++++++++--
 src/memes.ts       |  3 ++-
 src/scraper.ts     | 17 ++++++++---------
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/attachments.ts b/src/attachments.ts
index 5f70447..56f5537 100644
--- a/src/attachments.ts
+++ b/src/attachments.ts
@@ -2,6 +2,8 @@ import sharp from 'sharp'
 import axios from 'axios'
 import mime from 'mime-types'
 
+import _ from 'lodash';
+
 import type { Logger } from 'pino'
 
 import type Minio from 'minio'
@@ -53,7 +55,12 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
             return download.data
         }
 
-        return minio.putObject(STORAGE_BUCKET, objectName(a), stream())
+        return minio.putObject(STORAGE_BUCKET,
+            objectName(a),
+            stream(),
+            null,
+            // hardcode this for now, we upload everything as webp
+            { 'Content-Type': 'image/webp' })
     }
 
     async function deleteAttachment(a: Attachment) {
@@ -82,7 +89,7 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
             // when ppl reply to an attachment it is included in their message
             if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') {
                 log.error(e, 'failed to process attachment')
-                log.error(a, 'failed to process attachment')
+                log.error(_.omit(a, ['vision']), 'failed to process attachment')
             }
 
             // deletes the upload if we got that far
diff --git a/src/memes.ts b/src/memes.ts
index 8e1aada..67ff338 100644
--- a/src/memes.ts
+++ b/src/memes.ts
@@ -65,7 +65,8 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
 
     async function persist(m: Meme) {
         const index = meili.index('memes')
-        return index.updateDocuments([m])
+        const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
+        return index.updateDocuments([doc])
     }
 
     await ensureMeiliIndex();
diff --git a/src/scraper.ts b/src/scraper.ts
index 1b0b37c..89eb823 100644
--- a/src/scraper.ts
+++ b/src/scraper.ts
@@ -55,9 +55,7 @@ export default async function scraper(
     async function fetchGuildDetails(guildId) {
         const url = `https://discord.com/api/v9/guilds/${guildId}`
         const res = await axios.get(url, { headers })
-        const keys_to_remove = ['emojis', 'stickers', 'roles', 'features']
-        keys_to_remove.forEach(k => delete res.data[k])
-        return res.data
+        return _.omit(res.data, ['emojis', 'stickers', 'roles', 'features'])
     }
 
     async function fetchChannel(channelId) {
@@ -104,12 +102,11 @@ export default async function scraper(
                 .filter(m => m.attachments)
                 // bundle message and attachment together
                 .flatMap(m => m.attachments.map(a => [a, m]))
-                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
 
-            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
-            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+            log.info({ validAttachments: validAttachments.length }, 'attachments found in messages')
 
-            return Promise.all(newAttachments
+            return Promise.all(validAttachments
                 .map(async ([a, m]) => {
                     const attachment: Attachment = {
                         id: BigInt(a.id),
@@ -121,15 +118,17 @@ export default async function scraper(
                         // create this ahead of time to associate with our copy of the media
                         memeId: crypto.randomUUID(),
 
-                        // @ts-ignore it's fine
-                        vision: {},
+                        vision: {}, // quick hack so that detectText can receive a full attachment
                         guild,
                         channel,
                         message: m,
                         postedAt: convertSnowflakeToDate(a.id),
+                        // ts asks for these to be included but they don't need to be set
                         createdAt: undefined,
                         updatedAt: undefined
                     }
+
+                    // actually assign vision here
                     return { ...attachment, vision: await detectText(attachment) }
                 }))
         }