diff --git a/index.ts b/index.ts index b8ef5b9..a2dc46c 100644 --- a/index.ts +++ b/index.ts @@ -60,7 +60,6 @@ async function processChannels(channelIds) { return results } - async function updateMeili() { await ensureMeiliIndex(); const index = meili.index('memes') diff --git a/memestream.sublime-project b/memestream.sublime-project index 42968ee..f0891d8 100644 --- a/memestream.sublime-project +++ b/memestream.sublime-project @@ -10,5 +10,9 @@ "name": "Memestream - npm start", "cmd": ["npm", "start"], }, - ], + { + "name": "Memestream - npm test", + "cmd": ["npm", "test"], + } + ] } diff --git a/package-lock.json b/package-lock.json index 771f098..1af4978 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@prisma/client": "^5.12.1", "axios": "^1.6.8", "dotenv": "^16.4.1", + "lodash": "^4.17.21", "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", @@ -22,6 +23,7 @@ "typescript": "^5.3.3" }, "devDependencies": { + "@types/lodash": "^4.17.0", "esrun": "^3.2.26", "prisma": "^5.12.1" } @@ -1012,6 +1014,12 @@ "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" }, + "node_modules/@types/lodash": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", + "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", + "dev": true + }, "node_modules/@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", @@ -3462,6 +3470,12 @@ "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" }, + "@types/lodash": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", + "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", + "dev": true + }, "@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", diff --git a/package.json b/package.json index 19f7a9f..8e4eb9f 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "type": "module", "scripts": { "start": "esrun index.ts | pino-pretty", - "dev": "esrun --watch index.ts | | pino-pretty", + "dev": "esrun --watch index.ts | pino-pretty", "migrate:make": "prisma migrate dev --name", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "esrun src/scraper.test.ts | pino-pretty" }, "author": "", "license": "ISC", @@ -17,6 +17,7 @@ "@prisma/client": "^5.12.1", "axios": "^1.6.8", "dotenv": "^16.4.1", + "lodash": "^4.17.21", "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", @@ -26,6 +27,7 @@ "typescript": "^5.3.3" }, "devDependencies": { + "@types/lodash": "^4.17.0", "esrun": "^3.2.26", "prisma": "^5.12.1" } diff --git a/prisma/migrations/20240410091812_add_channelhistory/migration.sql b/prisma/migrations/20240410091812_add_channelhistory/migration.sql new file mode 100644 index 0000000..4aca86f --- /dev/null +++ b/prisma/migrations/20240410091812_add_channelhistory/migration.sql @@ -0,0 +1,9 @@ +-- CreateTable +CREATE TABLE "channelhistory" ( + "id" BIGINT NOT NULL, + "earliestMessageId" BIGINT NOT NULL, + "latestMessageId" BIGINT NOT NULL, + "backfillComplete" BOOLEAN NOT NULL DEFAULT false, + + CONSTRAINT "channelhistory_pkey" PRIMARY KEY ("id") +); diff --git a/prisma/migrations/20240410093508_drop_backfillcomplete/migration.sql b/prisma/migrations/20240410093508_drop_backfillcomplete/migration.sql new file mode 100644 index 0000000..50d88f5 --- /dev/null +++ b/prisma/migrations/20240410093508_drop_backfillcomplete/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the column `backfillComplete` on the `channelhistory` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "channelhistory" DROP COLUMN "backfillComplete"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 03e70bd..fec3e12 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -49,4 +49,11 @@ model Meme { @@map("memes") } +model ChannelHistory { + id BigInt @id + earliestMessageId BigInt + latestMessageId BigInt + // we do this because you have to always wrap table names in "" if they have capitals + @@map("channelhistory") +} diff --git a/src/scraper.test.ts b/src/scraper.test.ts new file mode 100644 index 0000000..1d08bb5 --- /dev/null +++ b/src/scraper.test.ts @@ -0,0 +1,44 @@ +import "dotenv/config" + +import fs from 'fs/promises' + +import pino from 'pino' + +import * as Minio from 'minio' +import Vision from '@google-cloud/vision' +import { PrismaClient } from '@prisma/client' +import { MeiliSearch } from 'meilisearch' + +import discordScraper from './scraper' +import attachmentService from "./attachments" + +// @ts-ignore - ts don't know wtf it talking about +BigInt.prototype.toJSON = function bigIntStringify() { + return this.toString(); +}; + +const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default +const STORAGE_BUCKET = process.env.STORAGE_BUCKET + +const log = pino({ base: {} }); +const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) +const prisma = new PrismaClient() +const minio = new Minio.Client({ + endPoint: process.env.STORAGE_URL, + accessKey: process.env.STORAGE_ACCESS_KEY, + secretKey: process.env.STORAGE_SECRET_KEY, +}) +const meili = new MeiliSearch({ + host: process.env.MEILI_URL, + apiKey: process.env.MEILI_MASTER_KEY, +}) + +const attachments = attachmentService(log, minio, prisma) +const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log) + +const CHANNEL_IDS = [ + 367936426962190336n, // TPM shitposts + 1095060717117374687n, // aella memes +] + +await Promise.all(CHANNEL_IDS.map(id => scraper.getChannelHistory(id))) diff --git a/src/scraper.ts b/src/scraper.ts index 207395b..7d5b46f 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -1,5 +1,7 @@ import axios from 'axios' +import minBy from 'lodash/minBy' + import type { PrismaClient, Attachment } from '@prisma/client'; import type { ImageAnnotatorClient } from '@google-cloud/vision'; import type { Logger } from 'pino'; @@ -20,6 +22,18 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio return res.data } + async function fetchChannelMessagesAfter(channelId, lastMessageId) { + const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}` + const res = await axios.get(url, { headers }) + return res.data + } + + async function fetchChannelMessagesBefore(channelId, earliestMessageId) { + const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}` + const res = await axios.get(url, { headers }) + return res.data + } + async function fetchGuildDetails(guildId) { const url = `https://discord.com/api/v9/guilds/${guildId}` const res = await axios.get(url, { headers }) @@ -64,12 +78,12 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio .flat() .filter(([a, m]) => acceptedContentTypes.includes(a.content_type)) - const attachmentIds = validAttachments.map(([a, m]) => a.id) - const alreadyProcessedIds = (await prisma.attachment - .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}}) - ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string + // const attachmentIds = validAttachments.map(([a, m]) => a.id) + // const alreadyProcessedIds = (await prisma.attachment + // .findMany({ select: { id: true }, where: { id: { in: attachmentIds }}}) + // ).map(r => r.id.toString()) // attachment not a true attachment yet, id still string - const newAttachments = validAttachments.filter(([a, m]) => !alreadyProcessedIds.includes(a.id)) + const newAttachments = validAttachments.filter(([a, m]) => ![].includes(a.id)) log.info({ newAttachments: newAttachments.length, validAttachments: validAttachments.length }) @@ -116,7 +130,72 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio } } + async function getChannelHistory(channelId) { + const channel = await fetchChannel(channelId) + const guild = await fetchGuildDetails(channel.guild_id) + const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId }) + + const ensureHistory = async () => { + const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }}) + if (stored) return stored + + const lastMessageId = channel.last_message_id + return await prisma.channelHistory.create({ data: + { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId } + }) + } + + async function fetchNew() { + const history = await ensureHistory() + const messages = await fetchChannelMessagesAfter(channelId, history.latestMessageId) + if (messages.length === 0) return + + const result = { + ...history, + messages: messages.length, + } + + const newLatestMessageId = messages.map(m => m.id) + log.info(newLatestMessageId) + } + + async function fetchOld() { + const history = await ensureHistory() + const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId) + if (messages.length === 0) return + + const result = { + ...history, + messages: messages.length, + } + + const earliestMessageId = minBy(messages, 'id').id + const attachments = await filterAttachments(log, messages, guild, channel); + + log.info({ + earliestMessageId, + messages: messages.length, + attachments: attachments.length, + }) + } + + await fetchNew() + await fetchOld() + + // process them + + // const messages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId); + + // const result = { + // ...history, + // newMessages: newMessages.length, + // messages: messages.length, + // } + // get messages before earliestMessageId until there are 0 + } + return { getChannelAttachments, + getChannelHistory, } }