diff --git a/index.ts b/index.ts index ab25bf0..ee6339e 100644 --- a/index.ts +++ b/index.ts @@ -37,16 +37,18 @@ const meili = new MeiliSearch({ const memes = await memeService(log, prisma, meili) const attachments = await attachmentService(log, minio, prisma, memes) -const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments) +const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments) const CHANNEL_IDS = [ 367936426962190336n, // TPM shitposts + 367936467592413194n, // tpm screenshots + 1028511952277864501n, // tpm sludge + 725835129326665768n, // machinegirl memes 1095060717117374687n, // aella memes ] async function main() { await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id))) - await memes.updateMeili(); } await main() diff --git a/search.html b/search.html deleted file mode 100644 index 4ef7cec..0000000 --- a/search.html +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - -
- - - - - - - diff --git a/src/attachments.ts b/src/attachments.ts index 49a5e8e..8f604d2 100644 --- a/src/attachments.ts +++ b/src/attachments.ts @@ -73,13 +73,14 @@ export default async function attachmentService(log: Logger, minio: Minio.Client await memes.persist(dbMeme) + log.info({ msg: 'new meme', dbMeme }) return { meme: dbMeme, upload, attachment: dbA } }, { timeout: 60000 }) } catch (e) { // this happens pretty regularly it seems // when ppl reply to an attachment it is included in their message - if (e.meta.modelName !== 'Meme' && e.meta.target[0] !== 'attachmentId') { + if (e.meta?.modelName !== 'Meme' && e.meta?.target[0] !== 'attachmentId') { log.error(e, 'failed to process attachment') log.error(a, 'failed to process attachment') } diff --git a/src/memes.ts b/src/memes.ts index bc3ddd1..2d7a6f9 100644 --- a/src/memes.ts +++ b/src/memes.ts @@ -2,6 +2,8 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client'; import type { Meilisearch } from 'meilisearch'; import type { Logger } from 'pino'; +import _ from 'lodash' + import * as attachments from './attachments' const STORAGE_BUCKET = process.env.STORAGE_BUCKET @@ -37,21 +39,24 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, } async function ensureMeiliIndex() { - const delTask = await meili.deleteIndex('memes') - const delResult = await meili.waitForTask(delTask.taskUid) const task = await meili.createIndex('memes', { primaryKey: 'id' }) const result = await meili.waitForTask(task.taskUid) log.debug(result); return meili.index('memes') } - async function updateMeili() { - await ensureMeiliIndex(); - const index = meili.index('memes') + // BLOWS IT UP + async function resetMeili() { + const delTask = await meili.deleteIndex('memes') + const delResult = await meili.waitForTask(delTask.taskUid) + const index = await ensureMeiliIndex() const memes = await prisma.meme.findMany(); - const task = await index.updateDocuments(memes) - const res = await meili.waitForTask(task.taskUid) - log.debug(res) + + for await (const batch of _.chunk(memes, 1000)) { + const task = await index.updateDocuments(batch) + const res = await meili.waitForTask(task.taskUid) + log.info({ res }, 'added meme batch') + } } async function persist(m: Meme) { @@ -60,11 +65,10 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient, } await ensureMeiliIndex(); - await updateMeili(); return { ensureMeiliIndex, - updateMeili, + resetMeili, fromAttachment, persist, } diff --git a/src/scraper.test.ts b/src/scraper.test.ts deleted file mode 100644 index fa289e3..0000000 --- a/src/scraper.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import "dotenv/config" - -import fs from 'fs/promises' - -import pino from 'pino' - -import * as Minio from 'minio' -import Vision from '@google-cloud/vision' -import { PrismaClient } from '@prisma/client' -import { MeiliSearch } from 'meilisearch' - -import discordScraper from './scraper' -import attachmentService from "./attachments" - -// @ts-ignore - ts don't know wtf it talking about -BigInt.prototype.toJSON = function bigIntStringify() { - return this.toString(); -}; - -const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default -const STORAGE_BUCKET = process.env.STORAGE_BUCKET - -const log = pino({ base: {} }); -const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) -const prisma = new PrismaClient() -const minio = new Minio.Client({ - endPoint: process.env.STORAGE_URL, - accessKey: process.env.STORAGE_ACCESS_KEY, - secretKey: process.env.STORAGE_SECRET_KEY, -}) -const meili = new MeiliSearch({ - host: process.env.MEILI_URL, - apiKey: process.env.MEILI_MASTER_KEY, -}) - -const attachments = attachmentService(log, minio, prisma) -const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log) - -const CHANNEL_IDS = [ - 367936426962190336n, // TPM shitposts - 1095060717117374687n, // aella memes -] - -await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id))) diff --git a/src/scraper.ts b/src/scraper.ts index 05d972c..4116104 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -15,7 +15,7 @@ import { IMAGE_TYPES, VIDEO_TYPES } from './attachments' // TODO // also fetch embeds and their descriptions -export default function scraper( +export default async function scraper( discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client, @@ -77,6 +77,12 @@ export default function scraper( return res[0] || {} } + async function fetchMissingVision() { + const attachments = await prisma.attachment + .findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } }}) + baseLog.info({attachments: attachments.length }, 'missing vision attachments') + } + async function processChannel(channelId) { const channel = await fetchChannel(channelId) const guild = await fetchGuildDetails(channel.guild_id) @@ -110,19 +116,18 @@ export default function scraper( memeId: crypto.randomUUID(), // @ts-ignore it's fine - vision: {}, // await detectText(attachment), + vision: {}, guild, channel, message: m, - createdAt: undefined, updatedAt: undefined } - return attachment + return { ...attachment, vision: await detectText(attachment) } })) } - async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator { + async function* fetchNew(channelId, log, latestMessageId): AsyncIterable { // reverse the messages because we want to yield them in chronological order // the api call returns them most recent first const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse() @@ -137,7 +142,7 @@ export default function scraper( yield* fetchNew(channelId, log, cursor) } - async function* fetchOld(channelId, log, earliestMessageId): AsyncGenerator { + async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable { const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId) if (messages.length === 0) { log.info('no new messages')