add content-type to uploads

docker-compose.yaml

@@ -12,15 +12,6 @@ services:
     volumes:
       - "./data/pg:/var/lib/postgresql/data"
 
-  redis:
-    image: redis:latest
-    ports:
-      - '6379:6379'
-    command: ['redis-server', '/usr/local/etc/redis/redis.conf']
-    volumes:
-      - "./data/redis:/data"
-      - "./redis.conf:/usr/local/etc/redis/redis.conf"
-
   meili:
     image: getmeili/meilisearch:v1.7
     ports:
@@ -41,18 +32,3 @@ services:
     environment:
       MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
       MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
-
-  # meilisync:
-  #   platform: linux/x86_64
-  #   image: long2ice/meilisync
-  #   volumes:
-  #     - ./config.yml:/meilisync/config.yml
-
-  # nginx:
-  #   image: nginx:latest
-  #   depends_on:
-  #     - 'api'
-  #   volumes:
-  #     - './nginx/localhost.nginx.conf:/etc/nginx/conf.d/default.conf'
-  #   ports:
-  #     - '80:80'

package.json

@@ -15,7 +15,6 @@
   "dependencies": {
     "@google-cloud/vision": "^4.2.0",
     "@prisma/client": "^5.12.1",
-    "@theinternetfolks/snowflake": "^1.3.0",
     "axios": "^1.6.8",
     "dotenv": "^16.4.1",
     "lodash": "^4.17.21",

redis.conf

@@ -1 +0,0 @@
-appendfsync everysec

src/attachments.ts

@@ -2,6 +2,8 @@ import sharp from 'sharp'
 import axios from 'axios'
 import mime from 'mime-types'
 
+import _ from 'lodash';
+
 import type { Logger } from 'pino'
 
 import type Minio from 'minio'
@@ -53,7 +55,12 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
             return download.data
         }
 
-        return minio.putObject(STORAGE_BUCKET, objectName(a), stream())
+        return minio.putObject(STORAGE_BUCKET,
+            objectName(a),
+            stream(),
+            null,
+            // hardcode this for now, we upload everything as webp
+            { 'Content-Type': 'image/webp' })
     }
 
     async function deleteAttachment(a: Attachment) {
@@ -82,7 +89,7 @@ export default async function attachmentService(log: Logger, minio: Minio.Client
             // when ppl reply to an attachment it is included in their message
             if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') {
                 log.error(e, 'failed to process attachment')
-                log.error(a, 'failed to process attachment')
+                log.error(_.omit(a, ['vision']), 'failed to process attachment')
             }
 
             // deletes the upload if we got that far

src/memes.ts

@@ -65,7 +65,8 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
 
     async function persist(m: Meme) {
         const index = meili.index('memes')
-        return index.updateDocuments([m])
+        const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
+        return index.updateDocuments([doc])
     }
 
     await ensureMeiliIndex();

src/scraper.ts

@@ -55,9 +55,7 @@ export default async function scraper(
     async function fetchGuildDetails(guildId) {
         const url = `https://discord.com/api/v9/guilds/${guildId}`
         const res = await axios.get(url, { headers })
-        const keys_to_remove = ['emojis', 'stickers', 'roles', 'features']
+        return _.omit(res.data, ['emojis', 'stickers', 'roles', 'features'])
-        keys_to_remove.forEach(k => delete res.data[k])
-        return res.data
     }
 
     async function fetchChannel(channelId) {
@@ -104,12 +102,11 @@ export default async function scraper(
                 .filter(m => m.attachments)
                 // bundle message and attachment together
                 .flatMap(m => m.attachments.map(a => [a, m]))
-                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
 
-            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
+            log.info({ validAttachments: validAttachments.length }, 'attachments found in messages')
-            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
 
-            return Promise.all(newAttachments
+            return Promise.all(validAttachments
                 .map(async ([a, m]) => {
                     const attachment: Attachment = {
                         id: BigInt(a.id),
@@ -121,15 +118,17 @@ export default async function scraper(
                         // create this ahead of time to associate with our copy of the media
                         memeId: crypto.randomUUID(),
 
-                        // @ts-ignore it's fine
+                        vision: {}, // quick hack so that detectText can receive a full attachment
-                        vision: {},
                         guild,
                         channel,
                         message: m,
                         postedAt: convertSnowflakeToDate(a.id),
+                        // ts asks for these to be included but they don't need to be set
                         createdAt: undefined,
                         updatedAt: undefined
                     }
+
+                    // actually assign vision here
                     return { ...attachment, vision: await detectText(attachment) }
                 }))
         }