fixes for local dev

archiver/TODO.md

@@ -2,7 +2,4 @@
 
 ## TODO
 
-[ ] Backfill last N images from a channel
-[ ] Set oldest_img_id for channel
-[ ]
 [ ] change downloader to url type

archiver/index.ts

@@ -46,11 +46,11 @@ async function main() {
     const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
 
     const CHANNEL_IDS = [
-        367936426962190336n,  // TPM shitposts
+        // 367936426962190336n,  // TPM shitposts
-        367936467592413194n,  // tpm screenshots
+        // 367936467592413194n,  // tpm screenshots
-        1028511952277864501n,  // tpm sludge
+        // 1028511952277864501n,  // tpm sludge
-        725835129326665768n, // machinegirl memes
+        // 725835129326665768n, // machinegirl memes
-        // 1095060717117374687n, // aella memes
+        1095060717117374687n, // aella memes
     ]
 
     async function migrateDb() {

archiver/src/attachments.ts

@@ -1,12 +1,11 @@
-import sharp from 'sharp'
-import axios from 'axios'
-import mime from 'mime-types'
-
 import _ from 'lodash';
-
+import axios from 'axios'
-import type { Logger } from 'pino'
+import fs from 'fs/promises'
+import mime from 'mime-types'
+import sharp from 'sharp'
 
 import type Minio from 'minio'
+import type { Logger } from 'pino'
 import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client'
 
 import type { MemeService } from './memes'
@@ -35,12 +34,14 @@ export function objectName(a: Attachment) {
     return `${a.memeId}.${mime.extension(a.contentType)}`
 }
 
-
 export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) {
+    const BUCKET_POLICY = await fs.readFile('./storage-bucket-policy.json', { encoding: 'utf8' })
+
     async function ensureStorageBucketExists() {
         const exists = await minio.bucketExists(STORAGE_BUCKET)
         if (exists) return true
         await minio.makeBucket(STORAGE_BUCKET)
+        await minio.setBucketPolicy(STORAGE_BUCKET, BUCKET_POLICY)
     }
 
     async function storeAttachment(a: Attachment) {

archiver/src/scraper.ts

@@ -192,12 +192,12 @@ export default async function discordScraper(
                 newCount++
             }
 
-            log.info({ oldCount, newCount }, 'processed attachments')
+            log.debug({ oldCount, newCount }, 'processed attachments')
 
             const shouldWait = newCount === 0 && oldCount === 0
             const PROCESS_DELAY_MS = 1000 * 60 // 1 minute
             if (shouldWait) {
-                log.info({ PROCESS_DELAY_MS }, 'waiting for new messages')
+                log.debug({ PROCESS_DELAY_MS }, 'waiting for new messages')
                 await setTimeout(PROCESS_DELAY_MS)
             }
 

docker-compose.yaml

@@ -1,33 +1,6 @@
 ---
 
 services:
-  pg:
-    image: postgres:latest
-    env_file:
-      - .env.pg
-    ports:
-      - '5432:5432'
-    volumes:
-      - "./data/pg:/var/lib/postgresql/data"
-
-  meili:
-    image: getmeili/meilisearch:v1.7
-    env_file:
-      - .env.meili
-    ports:
-      - '7700:7700'
-    volumes:
-      - './data/meili:/meili_data'
-
-  minio:
-    image: quay.io/minio/minio
-    ports:
-      - 9000:9000
-      - 9001:9001
-    command: ['minio', 'server', '/data', '--console-address', ':9001']
-    volumes:
-      - './data/minio:/data'
-
   archiver:
     image: registry.strix.systems/memestream-archiver
     command: npm run dev
@@ -47,3 +20,24 @@ services:
       - 3000:3000
     volumes:
       - './client:/usr/src/memestream-client'
+
+  pg:
+    image: postgres:latest
+    env_file:
+      - .env.pg
+    ports:
+      - '5432:5432'
+
+  meili:
+    image: getmeili/meilisearch:v1.7
+    env_file:
+      - .env.meili
+    ports:
+      - '7700:7700'
+
+  minio:
+    image: quay.io/minio/minio
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: ['minio', 'server', '/data', '--console-address', ':9001']