import "dotenv/config" import fs from 'fs/promises' import pino from 'pino' import * as Minio from 'minio' import Vision from '@google-cloud/vision' import { PrismaClient } from '@prisma/client' import { MeiliSearch } from 'meilisearch' import discordScraper from './src/scraper' import testAttachment from "./src/test.attachment" import attachmentService from "./src/attachments" // @ts-ignore - ts don't know wtf it talking about BigInt.prototype.toJSON = function bigIntStringify() { return this.toString(); }; const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default const STORAGE_BUCKET = process.env.STORAGE_BUCKET const log = pino({ base: {} }); const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) const prisma = new PrismaClient() const minio = new Minio.Client({ endPoint: process.env.STORAGE_URL, accessKey: process.env.STORAGE_ACCESS_KEY, secretKey: process.env.STORAGE_SECRET_KEY, }) const meili = new MeiliSearch({ host: process.env.MEILI_URL, apiKey: process.env.MEILI_MASTER_KEY, }) const attachments = attachmentService(log, minio, prisma) const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log) async function ensureStorageBucketExists() { const exists = await minio.bucketExists(STORAGE_BUCKET) if (exists) return true await minio.makeBucket(STORAGE_BUCKET) } async function ensureMeiliIndex() { const delTask = await meili.deleteIndex('memes') const delResult = await meili.waitForTask(delTask.taskUid) const task = await meili.createIndex('memes', { primaryKey: 'id' }) const result = await meili.waitForTask(task.taskUid) log.info(result); } async function processChannels(channelIds) { await ensureStorageBucketExists() const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c))) const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a))) log.info(results.map(r => r.status)); return results } async function updateMeili() { await ensureMeiliIndex(); const index = meili.index('memes') const memes = await prisma.meme.findMany(); const task = await index.updateDocuments(memes) const res = await meili.waitForTask(task.taskUid) log.info(res) } const CHANNEL_IDS = [ 367936426962190336n, // TPM shitposts 1095060717117374687n, // aella memes ] await processChannels(CHANNEL_IDS) await updateMeili()