thank fk

src/attachments.ts

@@ -2,6 +2,8 @@ import sharp from 'sharp';
 import axios from 'axios'
 import mime from 'mime-types'
 
+import type { Logger } from 'pino';
+
 import type Minio from 'minio'
 import type { PrismaClient, Attachment } from '@prisma/client';
 
@@ -31,7 +33,7 @@ export function objectName(a: Attachment) {
     return `${a.memeId}.${mime.extension(a.contentType)}`
 }
 
-export default function attachmentService(minio: Minio.Client, prisma: PrismaClient) {
+export default function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient) {
     async function storeAttachment(a: Attachment) {
         const download = await axios.get(a.url, { responseType: 'stream' })
 
@@ -56,10 +58,10 @@ export default function attachmentService(minio: Minio.Client, prisma: PrismaCli
         const upload = await storeAttachment(a)
         return prisma.$transaction(async prisma => {
             const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
-            const dbA = prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
-            console.log('pending addition to db', dbMeme)
+            const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
+            log.info({ msg: 'pending addition to db', dbMeme })
             const upload = await storeAttachment(a)
-            console.log('successfully uploaded', upload)
+            log.info({ msg: 'successfully uploaded', upload })
             return { meme: dbMeme, upload, attachment: dbA }
         })
     }

src/memes.ts

@@ -2,10 +2,11 @@ import type { PrismaClient, Attachment, Meme } from '@prisma/client';
 
 import * as attachments from './attachments'
 
+const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+
 function memeUrl(a: Attachment) {
     const STORAGE_URL = process.env.STORAGE_URL
-    const path = attachments.objectName(a)
-    const u = new URL(path, `https://${STORAGE_URL}`)
+    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
     return u.href
 }
 

src/scraper.ts

@@ -2,18 +2,14 @@ import axios from 'axios'
 
 import type { PrismaClient, Attachment } from '@prisma/client';
 import type { ImageAnnotatorClient } from '@google-cloud/vision';
+import type { Logger } from 'pino';
 import type Minio from 'minio'
 
 import sharp from 'sharp';
 
 import * as attachments from './attachments'
 
-export default function scraper(
-discordAuth,
-vision: ImageAnnotatorClient,
-minio: Minio.Client,
-prisma: PrismaClient,
-) {
+export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client, prisma: PrismaClient, baseLog: Logger) {
     const headers = {
         'Authorization': discordAuth,
     }
@@ -55,7 +51,7 @@ prisma: PrismaClient,
         return res[0] || {}
     }
 
-    function filterAttachments(messages, guild, channel) {
+    async function filterAttachments(log, messages, guild, channel) {
         const acceptedContentTypes = [
             ...attachments.IMAGE_TYPES,
             ...attachments.VIDEO_TYPES,
@@ -69,10 +65,15 @@ prisma: PrismaClient,
             .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
 
         const attachmentIds = validAttachments.map(([a, m]) => a.id)
+        const alreadyProcessedIds = (await prisma.attachment
+            .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}})
+        ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string
 
-        const alreadyProcessedIds = prisma.attachment.find
+        const newAttachments = validAttachments.filter(([a, m]) => !alreadyProcessedIds.includes(a.id))
 
-        return Promise.all(validAttachments
+        log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
+
+        return Promise.all(newAttachments
             .map(async ([a, m]) => {
                 const attachment: Attachment = {
                     id: BigInt(a.id),
@@ -105,14 +106,13 @@ prisma: PrismaClient,
             const guild = await fetchGuildDetails(channel.guild_id)
             const messages = await fetchChannelMessages(channelId)
 
-            return filterAttachments(messages, guild, channel);
+            const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
+
+            return filterAttachments(log, messages, guild, channel);
 
             // return { messages, channel, guild }
         } catch (e) {
-            console.error('scrape error', {
-                err: e.toJSON(),
-                channelId,
-            })
+            baseLog.error({ err: e.toJSON(), channelId, })
         }
     }