fixes for local dev

archiver/TODO.md

@@ -2,7 +2,4 @@
 
 ## TODO
 
-[ ] Backfill last N images from a channel
-[ ] Set oldest_img_id for channel
-[ ]
 [ ] change downloader to url type

archiver/index.ts

@@ -46,11 +46,11 @@ async function main() {
     const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
 
     const CHANNEL_IDS = [
-        367936426962190336n,  // TPM shitposts
-        367936467592413194n,  // tpm screenshots
-        1028511952277864501n,  // tpm sludge
-        725835129326665768n, // machinegirl memes
-        // 1095060717117374687n, // aella memes
+        // 367936426962190336n,  // TPM shitposts
+        // 367936467592413194n,  // tpm screenshots
+        // 1028511952277864501n,  // tpm sludge
+        // 725835129326665768n, // machinegirl memes
+        1095060717117374687n, // aella memes
     ]
 
     async function migrateDb() {

archiver/src/attachments.ts

@@ -1,12 +1,11 @@
-import sharp from 'sharp'
-import axios from 'axios'
-import mime from 'mime-types'
-
 import _ from 'lodash';
-
-import type { Logger } from 'pino'
+import axios from 'axios'
+import fs from 'fs/promises'
+import mime from 'mime-types'
+import sharp from 'sharp'
 
 import type Minio from 'minio'
+import type { Logger } from 'pino'
 import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client'
 
 import type { MemeService } from './memes'
@@ -35,12 +34,14 @@ export function objectName(a: Attachment) {
     return `${a.memeId}.${mime.extension(a.contentType)}`
 }
 
-
 export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) {
+    const BUCKET_POLICY = await fs.readFile('./storage-bucket-policy.json', { encoding: 'utf8' })
+
     async function ensureStorageBucketExists() {
         const exists = await minio.bucketExists(STORAGE_BUCKET)
         if (exists) return true
         await minio.makeBucket(STORAGE_BUCKET)
+        await minio.setBucketPolicy(STORAGE_BUCKET, BUCKET_POLICY)
     }
 
     async function storeAttachment(a: Attachment) {

archiver/src/scraper.ts

@@ -192,12 +192,12 @@ export default async function discordScraper(
                 newCount++
             }
 
-            log.info({ oldCount, newCount }, 'processed attachments')
+            log.debug({ oldCount, newCount }, 'processed attachments')
 
             const shouldWait = newCount === 0 && oldCount === 0
             const PROCESS_DELAY_MS = 1000 * 60 // 1 minute
             if (shouldWait) {
-                log.info({ PROCESS_DELAY_MS }, 'waiting for new messages')
+                log.debug({ PROCESS_DELAY_MS }, 'waiting for new messages')
                 await setTimeout(PROCESS_DELAY_MS)
             }