diff --git a/src/attachments.ts b/src/attachments.ts index 941b556..d9916ce 100644 --- a/src/attachments.ts +++ b/src/attachments.ts @@ -54,9 +54,8 @@ export default function attachmentService(log: Logger, minio: Minio.Client, pris } async function process(a: Attachment) { - const meme = memes.fromAttachment(a) - const upload = await storeAttachment(a) return prisma.$transaction(async prisma => { + const meme = memes.fromAttachment(a) const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } }) const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } }) log.info({ msg: 'pending addition to db', dbMeme }) diff --git a/src/scraper.ts b/src/scraper.ts index 3f408e5..7d89dfa 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -75,17 +75,10 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio const validAttachments = messages .filter(m => m.attachments) // bundle message and attachment together - .map(m => m.attachments.map(a => [a, m])) - .flat() + .flatMap(m => m.attachments.map(a => [a, m])) .filter(([a, m]) => acceptedContentTypes.includes(a.content_type)) - // const attachmentIds = validAttachments.map(([a, m]) => a.id) - // const alreadyProcessedIds = (await prisma.attachment - // .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}}) - // ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string - const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id)) - log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length }) return Promise.all(newAttachments @@ -139,9 +132,8 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio } log.info({ messages: messages.length, latestMessageId }, 'fetched new messages') - for (const msg of messages) { - yield msg - } + yield* messages + // not sure what order these are in // or if guaranteed const cursor = maxBy(messages, 'id').id @@ -149,23 +141,19 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio } async function* fetchOld(channelId, log, earliestMessageId) { - let cursor = earliestMessageId - - while (true) { - const messages = await fetchChannelMessagesBefore(channelId, cursor) - if (messages.length === 0) { - log.info('reached beginning of channel') - return - } - - log.info({ messages: messages.length, cursor }, 'fetched older messages') - for (const msg of messages) { - yield msg - } - // not sure what order these are in - // or if guaranteed - cursor = minBy(messages, 'id').id + const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId) + if (messages.length === 0) { + log.info('no new messages') + return } + + log.info({ messages: messages.length, earliestMessageId }, 'fetched older messages') + yield* messages + + // not sure what order these are in + // or if guaranteed + const cursor = minBy(messages, 'id').id + yield* fetchNew(channelId, log, cursor) } async function getChannelHistory(channelId) {