yield each

src/scraper.ts

@@ -82,27 +82,22 @@ export default function scraper(
         const guild = await fetchGuildDetails(channel.guild_id)
         const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
-        async function filterAttachments(messages): Promise<Attachment[]> {
+        async function* filterAttachments(message): AsyncGenerator<Attachment> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
                 ...VIDEO_TYPES,
             ]
 
-            const validAttachments = messages
-                .filter(m => m.attachments)
-                // bundle message and attachment together
-                .flatMap(m => m.attachments.map(a => [a, m]))
-                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+            if (!message.attachments) return null
 
-            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
-            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+            const validAttachments = message.attachments
+                .filter(a => acceptedContentTypes.includes(a.content_type))
 
-            return Promise.all(newAttachments
-                .map(async ([a, m]) => {
+            for (const a of validAttachments) {
                 const attachment: Attachment = {
                     id: BigInt(a.id),
-                        msgId: BigInt(m.id),
-                        channelId: BigInt(m.channel_id),
+                    msgId: BigInt(message.id),
+                    channelId: BigInt(message.channel_id),
                     url: a.url,
                     contentType: a.content_type,
 
@@ -110,16 +105,20 @@ export default function scraper(
                     memeId: crypto.randomUUID(),
 
                     // @ts-ignore it's fine
-                        vision: {}, // await detectText(attachment),
+                    vision: {},
                     guild,
                     channel,
-                        message: m,
+                    message,
 
                     createdAt: undefined,
                     updatedAt: undefined
                 }
-                    return attachment
-                }))
+
+                log.info('fetching vision')
+
+                const visionData = await detectText(attachment)
+                yield { ...attachment, vision: visionData }
+            }
         }
 
         async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator<Attachment> {
@@ -132,7 +131,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchNew(channelId, log, cursor)
         }
@@ -145,7 +147,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchOld(channelId, log, cursor)
         }