vision fix

src/memes.ts

@@ -2,6 +2,8 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client';
 import type { Meilisearch } from 'meilisearch';
 import type { Logger } from 'pino';
 
+import _ from 'lodash'
+
 import * as attachments from './attachments'
 
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
@@ -37,21 +39,24 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     async function ensureMeiliIndex() {
-        const delTask = await meili.deleteIndex('memes')
-        const delResult = await meili.waitForTask(delTask.taskUid)
         const task = await meili.createIndex('memes', { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
         return meili.index('memes')
     }
 
-    async function updateMeili() {
-        await ensureMeiliIndex();
-        const index = meili.index('memes')
+    // BLOWS IT UP
+    async function resetMeili() {
+        const delTask = await meili.deleteIndex('memes')
+        const delResult = await meili.waitForTask(delTask.taskUid)
+        const index = await ensureMeiliIndex()
         const memes = await prisma.meme.findMany();
-        const task = await index.updateDocuments(memes)
-        const res = await meili.waitForTask(task.taskUid)
-        log.debug(res)
+
+        for await (const batch of _.chunk(memes, 1000)) {
+            const task = await index.updateDocuments(batch)
+            const res = await meili.waitForTask(task.taskUid)
+            log.info({ res }, 'added meme batch')
+        }
     }
 
     async function persist(m: Meme) {
@@ -60,11 +65,10 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     await ensureMeiliIndex();
-    await updateMeili();
 
     return {
         ensureMeiliIndex,
-        updateMeili,
+        resetMeili,
         fromAttachment,
         persist,
     }