yield each

src/scraper.ts

@@ -82,44 +82,43 @@ export default function scraper(
         const guild = await fetchGuildDetails(channel.guild_id)
         const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
-        async function filterAttachments(messages): Promise<Attachment[]> {
+        async function* filterAttachments(message): AsyncGenerator<Attachment> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
                 ...VIDEO_TYPES,
             ]
 
-            const validAttachments = messages
+            if (!message.attachments) return null
-                .filter(m => m.attachments)
-                // bundle message and attachment together
-                .flatMap(m => m.attachments.map(a => [a, m]))
-                .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
 
-            const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
+            const validAttachments = message.attachments
-            log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+                .filter(a => acceptedContentTypes.includes(a.content_type))
 
-            return Promise.all(newAttachments
+            for (const a of validAttachments) {
-                .map(async ([a, m]) => {
+                const attachment: Attachment = {
-                    const attachment: Attachment = {
+                    id: BigInt(a.id),
-                        id: BigInt(a.id),
+                    msgId: BigInt(message.id),
-                        msgId: BigInt(m.id),
+                    channelId: BigInt(message.channel_id),
-                        channelId: BigInt(m.channel_id),
+                    url: a.url,
-                        url: a.url,
+                    contentType: a.content_type,
-                        contentType: a.content_type,
 
-                        // create this ahead of time to associate with our copy of the media
+                    // create this ahead of time to associate with our copy of the media
-                        memeId: crypto.randomUUID(),
+                    memeId: crypto.randomUUID(),
 
-                        // @ts-ignore it's fine
+                    // @ts-ignore it's fine
-                        vision: {}, // await detectText(attachment),
+                    vision: {},
-                        guild,
+                    guild,
-                        channel,
+                    channel,
-                        message: m,
+                    message,
 
-                        createdAt: undefined,
+                    createdAt: undefined,
-                        updatedAt: undefined
+                    updatedAt: undefined
-                    }
+                }
-                    return attachment
+
-                }))
+                log.info('fetching vision')
+
+                const visionData = await detectText(attachment)
+                yield { ...attachment, vision: visionData }
+            }
         }
 
         async function* fetchNew(channelId, log, latestMessageId): AsyncGenerator<Attachment> {
@@ -132,7 +131,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchNew(channelId, log, cursor)
         }
@@ -145,7 +147,10 @@ export default function scraper(
             }
 
             log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages')
-            yield* await filterAttachments(messages)
+            for (const msg of messages) {
+                yield* filterAttachments(msg)
+            }
+
             const cursor = _.last(messages).id
             yield* fetchOld(channelId, log, cursor)
         }