sublime fkt but everything working
This commit is contained in:
1 parent
b655adc305
commit
66228047f4
12 files changed
+363
-123
No files matched your search
@@ -1,40 +1,57 @@
|
||||
import "dotenv/config"
|
||||
|
||||
import fs from 'fs/promises'
|
||||
|
||||
import * as Minio from 'minio'
|
||||
import Vision from '@google-cloud/vision'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { MeiliSearch } from 'meilisearch'
|
||||
|
||||
import discordScraper from './src/scraper'
|
||||
import testAttachment from "./src/test.attachment"
|
||||
import attachmentService from "./src/attachments"
|
||||
|
||||
// @ts-ignore - ts don't know wtf it talking about
|
||||
BigInt.prototype.toJSON = function bigIntStringify() {
|
||||
return this.toString();
|
||||
};
|
||||
|
||||
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
|
||||
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
||||
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
||||
const prisma = new PrismaClient()
|
||||
const minio = new Minio.Client({
|
||||
endPoint: process.env.STORAGE_URL,
|
||||
accessKey: process.env.STORAGE_ACCESS_KEY,
|
||||
secretKey: process.env.STORAGE_SECRET_KEY,
|
||||
})
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
||||
const prisma = new PrismaClient()
|
||||
const client = new MeiliSearch({
|
||||
host: process.env.MEILI_URL,
|
||||
apiKey: process.env.MEILI_MASTER_KEY,
|
||||
})
|
||||
|
||||
async function storageBucketExists() {
|
||||
const attachments = attachmentService(minio, prisma)
|
||||
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
|
||||
|
||||
async function ensureStorageBucketExists() {
|
||||
const exists = await minio.bucketExists(STORAGE_BUCKET)
|
||||
if (exists) return true
|
||||
await minio.makeBucket(STORAGE_BUCKET)
|
||||
}
|
||||
|
||||
console.log(await storageBucketExists())
|
||||
async function processChannels(channelIds) {
|
||||
await ensureStorageBucketExists()
|
||||
const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c)))
|
||||
const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a)))
|
||||
console.log(results);
|
||||
return results
|
||||
}
|
||||
|
||||
// BIGINTS
|
||||
// must be strings
|
||||
const CHANNEL_IDS = [
|
||||
'367936426962190336', // TPM shitposts
|
||||
'1095060717117374687', // aella memes
|
||||
367936426962190336n, // TPM shitposts
|
||||
1095060717117374687n, // aella memes
|
||||
]
|
||||
|
||||
const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
|
||||
|
||||
const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c)))
|
||||
console.log(msgs)
|
||||
const downloads = await Promise.all(msgs.flat().map(scraper.downloadAttachment))
|
||||
await processChannels(CHANNEL_IDS)
|
||||
|
||||
Reference in new issue
Block a user