generatin

src/attachments.ts

@@ -54,9 +54,8 @@ export default function attachmentService(log: Logger, minio: Minio.Client, pris
     }
 
     async function process(a: Attachment) {
-        const meme = memes.fromAttachment(a)
-        const upload = await storeAttachment(a)
         return prisma.$transaction(async prisma => {
+            const meme = memes.fromAttachment(a)
             const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
             const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
             log.info({ msg: 'pending addition to db', dbMeme })

src/scraper.ts

@@ -75,17 +75,10 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         const validAttachments = messages
             .filter(m => m.attachments)
             // bundle message and attachment together
-            .map(m => m.attachments.map(a => [a, m]))
+            .flatMap(m => m.attachments.map(a => [a, m]))
-            .flat()
             .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
 
-        // const attachmentIds = validAttachments.map(([a, m]) => a.id)
-        // const alreadyProcessedIds = (await prisma.attachment
-        //     .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}})
-        // ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string
-
         const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
-
         log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
 
         return Promise.all(newAttachments
@@ -139,9 +132,8 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         }
 
         log.info({ messages: messages.length, latestMessageId }, 'fetched new messages')
-        for (const msg of messages) {
+        yield* messages
-            yield msg
+
-        }
         // not sure what order these are in
         // or if guaranteed
         const cursor = maxBy(messages, 'id').id
@@ -149,23 +141,19 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
     }
 
     async function* fetchOld(channelId, log, earliestMessageId) {
-        let cursor = earliestMessageId
+        const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
-
-        while (true) {
-            const messages = await fetchChannelMessagesBefore(channelId, cursor)
         if (messages.length === 0) {
-                log.info('reached beginning of channel')
+            log.info('no new messages')
             return
         }
 
-            log.info({ messages: messages.length, cursor }, 'fetched older messages')
+        log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages')
-            for (const msg of messages) {
+        yield* messages
-                yield msg
+
-            }
         // not sure what order these are in
         // or if guaranteed
-            cursor = minBy(messages, 'id').id
+        const cursor = minBy(messages, 'id').id
-        }
+        yield* fetchNew(channelId, log, cursor)
     }
 
     async function getChannelHistory(channelId) {