gnerator

.gitignore

@@ -2,6 +2,7 @@
 
 .venv/
 gimmeme-d052de2b9472.json
+gimmeme-db7225b3cf31.json
 __pycache__/
 
 node_modules

src/scraper.ts

@@ -1,6 +1,7 @@
 import axios from 'axios'
 
 import minBy from 'lodash/minBy'
+import maxBy from 'lodash/maxBy'
 
 import type { PrismaClient, Attachment } from '@prisma/client';
 import type { ImageAnnotatorClient } from '@google-cloud/vision';
@@ -130,6 +131,43 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         }
     }
 
+    async function* fetchNew(channelId, log, latestMessageId) {
+        const messages = await fetchChannelMessagesAfter(channelId, latestMessageId)
+        if (messages.length === 0) {
+            log.info('no new messages')
+            return
+        }
+
+        log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
+        for (const msg of messages) {
+            yield msg
+        }
+        // not sure what order these are in
+        // or if guaranteed
+        const cursor = maxBy(messages, 'id').id
+        yield* fetchNew(channelId, log, cursor)
+    }
+
+    async function* fetchOld(channelId, log, earliestMessageId) {
+        let cursor = earliestMessageId
+
+        while (true) {
+            const messages = await fetchChannelMessagesBefore(channelId, cursor)
+            if (messages.length === 0) {
+                log.info('reached beginning of channel')
+                return
+            }
+
+            log.info({ messages: messages.length, cursor }, 'fetched older messages')
+            for (const msg of messages) {
+                yield msg
+            }
+            // not sure what order these are in
+            // or if guaranteed
+            cursor = minBy(messages, 'id').id
+        }
+    }
+
     async function getChannelHistory(channelId) {
         const channel = await fetchChannel(channelId)
         const guild = await fetchGuildDetails(channel.guild_id)
@@ -145,53 +183,16 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
             })
         }
 
-        async function fetchNew() {
         const history = await ensureHistory()
-            const messages = await fetchChannelMessagesAfter(channelId, history.latestMessageId)
+        history.latestMessageId = 1227182587177340938n
-            if (messages.length === 0) return
 
-            const result = {
+        for await (const msg of fetchNew(channelId, log, history.latestMessageId)) {
-                ...history,
+            log.info({ msg: 'new message', id: msg.id})
-                messages: messages.length,
         }
 
-            const newLatestMessageId = messages.map(m => m.id)
+        // for await (const msg of fetchOld(channelId, log, history.earliestMessageId)) {
-            log.info(newLatestMessageId)
+        //     // log.info({ msg: 'new message', id: msg.id})
-        }
-
-        async function fetchOld() {
-            const history = await ensureHistory()
-            const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId)
-            if (messages.length === 0) return
-
-            const result = {
-                ...history,
-                messages: messages.length,
-            }
-
-            const earliestMessageId = minBy(messages, 'id').id
-            const attachments = await filterAttachments(log, messages, guild, channel);
-
-            log.info({
-                earliestMessageId,
-                messages: messages.length,
-                attachments: attachments.length,
-            })
-        }
-
-        await fetchNew()
-        await fetchOld()
-
-        // process them
-
-        // const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId);
-
-        // const result = {
-        //     ...history,
-        //     newMessages: newMessages.length,
-        //     messages: messages.length,
         // }
-        // get messages before earliestMessageId until there are 0
     }
 
     return {