58 lines
1.8 KiB
TypeScript
58 lines
1.8 KiB
TypeScript
import "dotenv/config"
|
|
|
|
import fs from 'fs/promises'
|
|
|
|
import * as Minio from 'minio'
|
|
import Vision from '@google-cloud/vision'
|
|
import { PrismaClient } from '@prisma/client'
|
|
import { MeiliSearch } from 'meilisearch'
|
|
|
|
import discordScraper from './src/scraper'
|
|
import testAttachment from "./src/test.attachment"
|
|
import attachmentService from "./src/attachments"
|
|
|
|
// @ts-ignore - ts don't know wtf it talking about
|
|
BigInt.prototype.toJSON = function bigIntStringify() {
|
|
return this.toString();
|
|
};
|
|
|
|
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
|
|
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
|
|
|
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
|
const prisma = new PrismaClient()
|
|
const minio = new Minio.Client({
|
|
endPoint: process.env.STORAGE_URL,
|
|
accessKey: process.env.STORAGE_ACCESS_KEY,
|
|
secretKey: process.env.STORAGE_SECRET_KEY,
|
|
})
|
|
const client = new MeiliSearch({
|
|
host: process.env.MEILI_URL,
|
|
apiKey: process.env.MEILI_MASTER_KEY,
|
|
})
|
|
|
|
const attachments = attachmentService(minio, prisma)
|
|
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
|
|
|
|
async function ensureStorageBucketExists() {
|
|
const exists = await minio.bucketExists(STORAGE_BUCKET)
|
|
if (exists) return true
|
|
await minio.makeBucket(STORAGE_BUCKET)
|
|
}
|
|
|
|
async function processChannels(channelIds) {
|
|
await ensureStorageBucketExists()
|
|
const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c)))
|
|
const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a)))
|
|
console.log(results);
|
|
return results
|
|
}
|
|
|
|
const CHANNEL_IDS = [
|
|
367936426962190336n, // TPM shitposts
|
|
1095060717117374687n, // aella memes
|
|
]
|
|
|
|
await processChannels(CHANNEL_IDS)
|
|
|