81 lines
2.5 KiB
TypeScript
81 lines
2.5 KiB
TypeScript
import "dotenv/config"
|
|
|
|
import fs from 'fs/promises'
|
|
|
|
import pino from 'pino'
|
|
|
|
import * as Minio from 'minio'
|
|
import Vision from '@google-cloud/vision'
|
|
import { PrismaClient } from '@prisma/client'
|
|
import { MeiliSearch } from 'meilisearch'
|
|
|
|
import discordScraper from './src/scraper'
|
|
import testAttachment from "./src/test.attachment"
|
|
import attachmentService from "./src/attachments"
|
|
|
|
// @ts-ignore - ts don't know wtf it talking about
|
|
BigInt.prototype.toJSON = function bigIntStringify() {
|
|
return this.toString();
|
|
};
|
|
|
|
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
|
|
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
|
|
|
const log = pino({ base: {} });
|
|
|
|
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
|
const prisma = new PrismaClient()
|
|
const minio = new Minio.Client({
|
|
endPoint: process.env.STORAGE_URL,
|
|
accessKey: process.env.STORAGE_ACCESS_KEY,
|
|
secretKey: process.env.STORAGE_SECRET_KEY,
|
|
})
|
|
const meili = new MeiliSearch({
|
|
host: process.env.MEILI_URL,
|
|
apiKey: process.env.MEILI_MASTER_KEY,
|
|
})
|
|
|
|
const attachments = attachmentService(log, minio, prisma)
|
|
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log)
|
|
|
|
async function ensureStorageBucketExists() {
|
|
const exists = await minio.bucketExists(STORAGE_BUCKET)
|
|
if (exists) return true
|
|
await minio.makeBucket(STORAGE_BUCKET)
|
|
}
|
|
|
|
async function ensureMeiliIndex() {
|
|
const delTask = await meili.deleteIndex('memes')
|
|
const delResult = await meili.waitForTask(delTask.taskUid)
|
|
const task = await meili.createIndex('memes', { primaryKey: 'id' })
|
|
const result = await meili.waitForTask(task.taskUid)
|
|
log.info(result);
|
|
}
|
|
|
|
async function processChannels(channelIds) {
|
|
await ensureStorageBucketExists()
|
|
const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c)))
|
|
const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a)))
|
|
log.info(results.map(r => r.status));
|
|
return results
|
|
}
|
|
|
|
|
|
async function updateMeili() {
|
|
await ensureMeiliIndex();
|
|
const index = meili.index('memes')
|
|
const memes = await prisma.meme.findMany();
|
|
const task = await index.updateDocuments(memes)
|
|
const res = await meili.waitForTask(task.taskUid)
|
|
log.info(res)
|
|
}
|
|
|
|
const CHANNEL_IDS = [
|
|
367936426962190336n, // TPM shitposts
|
|
1095060717117374687n, // aella memes
|
|
]
|
|
|
|
await processChannels(CHANNEL_IDS)
|
|
await updateMeili()
|
|
|