From 8212f9bbf3d3e814b6108ca55a9f046128b303b4 Mon Sep 17 00:00:00 2001
From: Nathan Rashleigh <ntr@strix.systems>
Date: Fri, 12 Apr 2024 17:24:39 +1000
Subject: [PATCH] yield each

---
 src/scraper.ts | 65 +++++++++++++++++++++++++++-----------------------
 1 file changed, 35 insertions(+), 30 deletions(-)

diff --git a/src/scraper.ts b/src/scraper.ts
index 05d972c..4be23bb 100644
--- a/src/scraper.ts
+++ b/src/scraper.ts
@@ -82,44 +82,43 @@ export default function scraper(
         const guild = await fetchGuildDetails(channel.guild_id)
         const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
-        async function filterAttachments(messages): Promise<Attachment[]> {
+        async function* filterAttachments(message): AsyncGenerator<Attachment> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
                 ...VIDEO_TYPES,
             ]
 
-            const validAttachments = messages
-                .filter(m => m.attachments)
-                // bundle message and attachment together
-                .flatMap(m => m.attachments.map(a => [a, m]))
-                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+            if (!message.attachments) return null
 
-            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
-            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+            const validAttachments = message.attachments
+                .filter(a => acceptedContentTypes.includes(a.content_type))
 
-            return Promise.all(newAttachments
-                .map(async ([a, m]) => {
-                    const attachment: Attachment = {
-                        id: BigInt(a.id),
-                        msgId: BigInt(m.id),
-                        channelId: BigInt(m.channel_id),
-                        url: a.url,
-                        contentType: a.content_type,
+            for (const a of validAttachments) {
+                const attachment: Attachment = {
+                    id: BigInt(a.id),
+                    msgId: BigInt(message.id),
+                    channelId: BigInt(message.channel_id),
+                    url: a.url,
+                    contentType: a.content_type,
 
-                        // create this ahead of time to associate with our copy of the media
-                        memeId: crypto.randomUUID(),
+                    // create this ahead of time to associate with our copy of the media
+                    memeId: crypto.randomUUID(),
 
-                        // @ts-ignore it's fine
-                        vision: {}, // await detectText(attachment),
-                        guild,
-                        channel,
-                        message: m,
+                    // @ts-ignore it's fine
+                    vision: {},
+                    guild,
+                    channel,
+                    message,
 
-                        createdAt: undefined,
-                        updatedAt: undefined
-                    }
-                    return attachment
-                }))
+                    createdAt: undefined,
+                    updatedAt: undefined
+                }
+
+                log.info('fetching vision')
+
+                const visionData = await detectText(attachment)
+                yield { ...attachment, vision: visionData }
+            }
         }
 
         async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator<Attachment> {
@@ -132,7 +131,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchNew(channelId, log, cursor)
         }
@@ -145,7 +147,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchOld(channelId, log, cursor)
         }