diff --git a/archiver/TODO.md b/archiver/TODO.md index 2820fb4..ed3dfe7 100644 --- a/archiver/TODO.md +++ b/archiver/TODO.md @@ -2,7 +2,4 @@ ## TODO -[ ] Backfill last N images from a channel -[ ] Set oldest_img_id for channel -[ ] [ ] change downloader to url type diff --git a/archiver/index.ts b/archiver/index.ts index 6289dfe..f312d57 100644 --- a/archiver/index.ts +++ b/archiver/index.ts @@ -46,11 +46,11 @@ async function main() { const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments) const CHANNEL_IDS = [ - 367936426962190336n, // TPM shitposts - 367936467592413194n, // tpm screenshots - 1028511952277864501n, // tpm sludge - 725835129326665768n, // machinegirl memes - // 1095060717117374687n, // aella memes + // 367936426962190336n, // TPM shitposts + // 367936467592413194n, // tpm screenshots + // 1028511952277864501n, // tpm sludge + // 725835129326665768n, // machinegirl memes + 1095060717117374687n, // aella memes ] async function migrateDb() { diff --git a/archiver/src/attachments.ts b/archiver/src/attachments.ts index 72e5583..628ea99 100644 --- a/archiver/src/attachments.ts +++ b/archiver/src/attachments.ts @@ -1,12 +1,11 @@ -import sharp from 'sharp' -import axios from 'axios' -import mime from 'mime-types' - import _ from 'lodash'; - -import type { Logger } from 'pino' +import axios from 'axios' +import fs from 'fs/promises' +import mime from 'mime-types' +import sharp from 'sharp' import type Minio from 'minio' +import type { Logger } from 'pino' import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client' import type { MemeService } from './memes' @@ -35,12 +34,14 @@ export function objectName(a: Attachment) { return `${a.memeId}.${mime.extension(a.contentType)}` } - export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) { + const BUCKET_POLICY = await fs.readFile('./storage-bucket-policy.json', { encoding: 'utf8' }) + async function ensureStorageBucketExists() { const exists = await minio.bucketExists(STORAGE_BUCKET) if (exists) return true await minio.makeBucket(STORAGE_BUCKET) + await minio.setBucketPolicy(STORAGE_BUCKET, BUCKET_POLICY) } async function storeAttachment(a: Attachment) { diff --git a/archiver/src/scraper.ts b/archiver/src/scraper.ts index c5f6646..4047f24 100644 --- a/archiver/src/scraper.ts +++ b/archiver/src/scraper.ts @@ -192,12 +192,12 @@ export default async function discordScraper( newCount++ } - log.info({ oldCount, newCount }, 'processed attachments') + log.debug({ oldCount, newCount }, 'processed attachments') const shouldWait = newCount === 0 && oldCount === 0 const PROCESS_DELAY_MS = 1000 * 60 // 1 minute if (shouldWait) { - log.info({ PROCESS_DELAY_MS }, 'waiting for new messages') + log.debug({ PROCESS_DELAY_MS }, 'waiting for new messages') await setTimeout(PROCESS_DELAY_MS) } diff --git a/docker-compose.yaml b/docker-compose.yaml index e4ae212..49a8807 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,33 +1,6 @@ --- services: - pg: - image: postgres:latest - env_file: - - .env.pg - ports: - - '5432:5432' - volumes: - - "./data/pg:/var/lib/postgresql/data" - - meili: - image: getmeili/meilisearch:v1.7 - env_file: - - .env.meili - ports: - - '7700:7700' - volumes: - - './data/meili:/meili_data' - - minio: - image: quay.io/minio/minio - ports: - - 9000:9000 - - 9001:9001 - command: ['minio', 'server', '/data', '--console-address', ':9001'] - volumes: - - './data/minio:/data' - archiver: image: registry.strix.systems/memestream-archiver command: npm run dev @@ -47,3 +20,24 @@ services: - 3000:3000 volumes: - './client:/usr/src/memestream-client' + + pg: + image: postgres:latest + env_file: + - .env.pg + ports: + - '5432:5432' + + meili: + image: getmeili/meilisearch:v1.7 + env_file: + - .env.meili + ports: + - '7700:7700' + + minio: + image: quay.io/minio/minio + ports: + - 9000:9000 + - 9001:9001 + command: ['minio', 'server', '/data', '--console-address', ':9001']