scraping backwards

index.ts

@@ -12,6 +12,7 @@ import { MeiliSearch } from 'meilisearch'
 import discordScraper from './src/scraper'
 import testAttachment from "./src/test.attachment"
 import attachmentService from "./src/attachments"
+import memeService from "./src/memes"
 
 // @ts-ignore - ts don't know wtf it talking about
 BigInt.prototype.toJSON = function bigIntStringify() {
@@ -19,7 +20,6 @@ BigInt.prototype.toJSON = function bigIntStringify() {
 };
 
 const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
 
 const log = pino({ base: {} });
 
@@ -35,45 +35,20 @@ const meili = new MeiliSearch({
   apiKey: process.env.MEILI_MASTER_KEY,
 })
 
-const attachments = attachmentService(log, minio, prisma)
-const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log)
-
-async function ensureStorageBucketExists() {
-    const exists = await minio.bucketExists(STORAGE_BUCKET)
-    if (exists) return true
-    await minio.makeBucket(STORAGE_BUCKET)
-}
-
-async function ensureMeiliIndex() {
-    const delTask = await meili.deleteIndex('memes')
-    const delResult = await meili.waitForTask(delTask.taskUid)
-    const task = await meili.createIndex('memes', { primaryKey: 'id' })
-    const result = await meili.waitForTask(task.taskUid)
-    log.info(result);
-}
-
-async function processChannels(channelIds) {
-    await ensureStorageBucketExists()
-    const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c)))
-    const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a)))
-    log.info(results.map(r => r.status));
-    return results
-}
-
-async function updateMeili() {
-    await ensureMeiliIndex();
-    const index = meili.index('memes')
-    const memes = await prisma.meme.findMany();
-    const task = await index.updateDocuments(memes)
-    const res = await meili.waitForTask(task.taskUid)
-    log.info(res)
-}
+const memes = await memeService(log, prisma, meili)
+const attachments = await attachmentService(log, minio, prisma, memes)
+const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
 
 const CHANNEL_IDS = [
     367936426962190336n,  // TPM shitposts
     1095060717117374687n, // aella memes
 ]
 
-await processChannels(CHANNEL_IDS)
-await updateMeili()
+async function main() {
+    await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
+    await memes.updateMeili();
+}
+
+await main()
+