scroll?

index.ts

@@ -60,7 +60,6 @@ async function processChannels(channelIds) {
     return results
 }
 
-
 async function updateMeili() {
     await ensureMeiliIndex();
     const index = meili.index('memes')

memestream.sublime-project

@@ -10,5 +10,9 @@
             "name": "Memestream - npm start",
             "cmd": ["npm", "start"],
         },
-    ],
+        {
+            "name": "Memestream - npm test",
+            "cmd": ["npm", "test"],
+        }
+    ]
 }

package-lock.json

@@ -13,6 +13,7 @@
         "@prisma/client": "^5.12.1",
         "axios": "^1.6.8",
         "dotenv": "^16.4.1",
+        "lodash": "^4.17.21",
         "meilisearch": "^0.38.0",
         "mime-types": "^2.1.35",
         "minio": "^7.1.3",
@@ -22,6 +23,7 @@
         "typescript": "^5.3.3"
       },
       "devDependencies": {
+        "@types/lodash": "^4.17.0",
         "esrun": "^3.2.26",
         "prisma": "^5.12.1"
       }
@@ -1012,6 +1014,12 @@
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz",
       "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="
     },
+    "node_modules/@types/lodash": {
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
+      "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==",
+      "dev": true
+    },
     "node_modules/@types/long": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
@@ -3462,6 +3470,12 @@
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz",
       "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="
     },
+    "@types/lodash": {
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
+      "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==",
+      "dev": true
+    },
     "@types/long": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",

package.json

@@ -6,9 +6,9 @@
   "type": "module",
   "scripts": {
     "start": "esrun index.ts | pino-pretty",
-    "dev": "esrun --watch index.ts | | pino-pretty",
+    "dev": "esrun --watch index.ts | pino-pretty",
     "migrate:make": "prisma migrate dev --name",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "esrun src/scraper.test.ts | pino-pretty"
   },
   "author": "",
   "license": "ISC",
@@ -17,6 +17,7 @@
     "@prisma/client": "^5.12.1",
     "axios": "^1.6.8",
     "dotenv": "^16.4.1",
+    "lodash": "^4.17.21",
     "meilisearch": "^0.38.0",
     "mime-types": "^2.1.35",
     "minio": "^7.1.3",
@@ -26,6 +27,7 @@
     "typescript": "^5.3.3"
   },
   "devDependencies": {
+    "@types/lodash": "^4.17.0",
     "esrun": "^3.2.26",
     "prisma": "^5.12.1"
   }

prisma/migrations/20240410091812_add_channelhistory/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE "channelhistory" (
+    "id" BIGINT NOT NULL,
+    "earliestMessageId" BIGINT NOT NULL,
+    "latestMessageId" BIGINT NOT NULL,
+    "backfillComplete" BOOLEAN NOT NULL DEFAULT false,
+
+    CONSTRAINT "channelhistory_pkey" PRIMARY KEY ("id")
+);

prisma/migrations/20240410093508_drop_backfillcomplete/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `backfillComplete` on the `channelhistory` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "channelhistory" DROP COLUMN "backfillComplete";

prisma/schema.prisma

@@ -49,4 +49,11 @@ model Meme {
     @@map("memes")
 }
 
+model ChannelHistory {
+    id                  BigInt    @id
+    earliestMessageId   BigInt
+    latestMessageId     BigInt
 
+    // we do this because you have to always wrap table names in "" if they have capitals
+    @@map("channelhistory")
+}

src/scraper.test.ts

@@ -0,0 +1,44 @@
+import "dotenv/config"
+
+import fs from 'fs/promises'
+
+import pino from 'pino'
+
+import * as Minio from 'minio'
+import Vision from '@google-cloud/vision'
+import { PrismaClient } from '@prisma/client'
+import { MeiliSearch } from 'meilisearch'
+
+import discordScraper from './scraper'
+import attachmentService from "./attachments"
+
+// @ts-ignore - ts don't know wtf it talking about
+BigInt.prototype.toJSON = function bigIntStringify() {
+    return this.toString();
+};
+
+const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
+const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+
+const log = pino({ base: {} });
+const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
+const prisma = new PrismaClient()
+const minio = new Minio.Client({
+    endPoint: process.env.STORAGE_URL,
+    accessKey: process.env.STORAGE_ACCESS_KEY,
+    secretKey: process.env.STORAGE_SECRET_KEY,
+})
+const meili = new MeiliSearch({
+  host: process.env.MEILI_URL,
+  apiKey: process.env.MEILI_MASTER_KEY,
+})
+
+const attachments = attachmentService(log, minio, prisma)
+const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log)
+
+const CHANNEL_IDS = [
+    367936426962190336n,  // TPM shitposts
+    1095060717117374687n, // aella memes
+]
+
+await Promise.all(CHANNEL_IDS.map(id => scraper.getChannelHistory(id)))

src/scraper.ts

@@ -1,5 +1,7 @@
 import axios from 'axios'
 
+import minBy from 'lodash/minBy'
+
 import type { PrismaClient, Attachment } from '@prisma/client';
 import type { ImageAnnotatorClient } from '@google-cloud/vision';
 import type { Logger } from 'pino';
@@ -20,6 +22,18 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         return res.data
     }
 
+    async function fetchChannelMessagesAfter(channelId, lastMessageId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
+    async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
+        const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}`
+        const res = await axios.get(url, { headers })
+        return res.data
+    }
+
     async function fetchGuildDetails(guildId) {
         const url = `https://discord.com/api/v9/guilds/${guildId}`
         const res = await axios.get(url, { headers })
@@ -64,12 +78,12 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
             .flat()
             .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
 
-        const attachmentIds = validAttachments.map(([a, m]) => a.id)
-        const alreadyProcessedIds = (await prisma.attachment
-            .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}})
-        ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string
+        // const attachmentIds = validAttachments.map(([a, m]) => a.id)
+        // const alreadyProcessedIds = (await prisma.attachment
+        //     .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}})
+        // ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string
 
-        const newAttachments = validAttachments.filter(([a, m]) => !alreadyProcessedIds.includes(a.id))
+        const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id))
 
         log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length })
 
@@ -116,7 +130,72 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         }
     }
 
+    async function getChannelHistory(channelId) {
+        const channel = await fetchChannel(channelId)
+        const guild = await fetchGuildDetails(channel.guild_id)
+        const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
+
+        const ensureHistory = async () => {
+            const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }})
+            if (stored) return stored
+
+            const lastMessageId = channel.last_message_id
+            return await prisma.channelHistory.create({ data:
+                { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
+            })
+        }
+
+        async function fetchNew() {
+            const history = await ensureHistory()
+            const messages = await fetchChannelMessagesAfter(channelId, history.latestMessageId)
+            if (messages.length === 0) return
+
+            const result = {
+                ...history,
+                messages: messages.length,
+            }
+
+            const newLatestMessageId = messages.map(m => m.id)
+            log.info(newLatestMessageId)
+        }
+
+        async function fetchOld() {
+            const history = await ensureHistory()
+            const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId)
+            if (messages.length === 0) return
+
+            const result = {
+                ...history,
+                messages: messages.length,
+            }
+
+            const earliestMessageId = minBy(messages, 'id').id
+            const attachments = await filterAttachments(log, messages, guild, channel);
+
+            log.info({
+                earliestMessageId,
+                messages: messages.length,
+                attachments: attachments.length,
+            })
+        }
+
+        await fetchNew()
+        await fetchOld()
+
+        // process them
+
+        // const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId);
+
+        // const result = {
+        //     ...history,
+        //     newMessages: newMessages.length,
+        //     messages: messages.length,
+        // }
+        // get messages before earliestMessageId until there are 0
+    }
+
     return {
         getChannelAttachments,
+        getChannelHistory,
     }
 }