This commit is contained in:
Nathan Rashleigh committed 2024-06-30 23:19:06 +10:00
1 parent c987ba01b0
commit 9931732ded
2 files changed
+4 -14

No files matched your search

+3 -9
View File
@@ -1,6 +1,5 @@
import "dotenv/config" import "dotenv/config"
import fs from 'fs/promises'
import { exec as execCb } from 'child_process' import { exec as execCb } from 'child_process'
import { promisify } from 'util' import { promisify } from 'util'
@@ -21,9 +20,9 @@ BigInt.prototype.toJSON = function bigIntStringify() {
}; };
const exec = promisify(execCb) const exec = promisify(execCb)
const log = pino({ base: {} });
async function main() { async function main() {
const log = pino({ base: {} });
const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS) const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS)
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
@@ -43,7 +42,7 @@ async function main() {
const memes = await memeService(log, prisma, meili) const memes = await memeService(log, prisma, meili)
const attachments = await attachmentService(log, minio, prisma, memes) const attachments = await attachmentService(log, minio, prisma, memes)
const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments) const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, prisma, log, attachments)
const CHANNEL_IDS = [ const CHANNEL_IDS = [
// 367936426962190336n, // TPM shitposts // 367936426962190336n, // TPM shitposts
@@ -59,14 +58,9 @@ async function main() {
} }
await migrateDb() await migrateDb()
log.info('ready')
await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id))) await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
} }
main() main()
.then(() => true) .then(() => true)
.catch(e => console.error(e)) .catch(e => log.error(e))
+1 -5
View File
@@ -8,8 +8,6 @@ import type { ImageAnnotatorClient } from '@google-cloud/vision';
import type { Logger } from 'pino'; import type { Logger } from 'pino';
import type Minio from 'minio' import type Minio from 'minio'
import sharp from 'sharp';
import type { AttachmentService } from './attachments' import type { AttachmentService } from './attachments'
import { IMAGE_TYPES, VIDEO_TYPES } from './attachments' import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
@@ -25,7 +23,6 @@ function convertSnowflakeToDate(snowflake) {
export default async function discordScraper( export default async function discordScraper(
discordAuth, discordAuth,
vision: ImageAnnotatorClient, vision: ImageAnnotatorClient,
minio: Minio.Client,
prisma: PrismaClient, prisma: PrismaClient,
baseLog: Logger, baseLog: Logger,
attachments: AttachmentService attachments: AttachmentService
@@ -130,6 +127,7 @@ export default async function discordScraper(
} }
// actually assign vision here // actually assign vision here
// so we can pass an Attachment to detectText
return { ...attachment, vision: await detectText(attachment) } return { ...attachment, vision: await detectText(attachment) }
})) }))
} }
@@ -166,8 +164,6 @@ export default async function discordScraper(
}) })
} }
const history = await ensureHistory()
async function processLoop() { async function processLoop() {
const history = await ensureHistory() const history = await ensureHistory()
const newAttachments = fetchNew(channelId, log, history.latestMessageId) const newAttachments = fetchNew(channelId, log, history.latestMessageId)