345 lines
12 KiB
JavaScript
345 lines
12 KiB
JavaScript
// index.ts
|
|
import "dotenv/config";
|
|
import { exec as execCb } from "child_process";
|
|
import { promisify } from "util";
|
|
import pino from "pino";
|
|
import * as Minio from "minio";
|
|
import Vision from "@google-cloud/vision";
|
|
import { PrismaClient } from "@prisma/client";
|
|
import { MeiliSearch } from "meilisearch";
|
|
|
|
// src/scraper.ts
|
|
import { setTimeout } from "timers/promises";
|
|
import axios2 from "axios";
|
|
import _2 from "lodash";
|
|
|
|
// src/attachments.ts
|
|
import sharp from "sharp";
|
|
import axios from "axios";
|
|
import mime from "mime-types";
|
|
import _ from "lodash";
|
|
var STORAGE_BUCKET = process.env.STORAGE_BUCKET;
|
|
var IMAGE_TYPES = [
|
|
"image/png",
|
|
"image/jpeg",
|
|
"image/webp",
|
|
"image/gif"
|
|
];
|
|
function isImage(a) {
|
|
return IMAGE_TYPES.includes(a.contentType);
|
|
}
|
|
function objectName(a) {
|
|
if (isImage(a))
|
|
return `${a.memeId}.webp`;
|
|
return `${a.memeId}.${mime.extension(a.contentType)}`;
|
|
}
|
|
async function attachmentService(log, minio, prisma, memes) {
|
|
async function ensureStorageBucketExists() {
|
|
const exists = await minio.bucketExists(STORAGE_BUCKET);
|
|
if (exists)
|
|
return true;
|
|
await minio.makeBucket(STORAGE_BUCKET);
|
|
}
|
|
async function storeAttachment(a) {
|
|
const download = await axios.get(a.url, { responseType: "stream" });
|
|
const stream = () => {
|
|
if (isImage(a)) {
|
|
const converter = sharp().webp();
|
|
download.data.pipe(converter);
|
|
return converter;
|
|
}
|
|
return download.data;
|
|
};
|
|
return minio.putObject(
|
|
STORAGE_BUCKET,
|
|
objectName(a),
|
|
stream(),
|
|
null,
|
|
// hardcode this for now, we upload everything as webp
|
|
{ "Content-Type": "image/webp" }
|
|
);
|
|
}
|
|
async function deleteAttachment(a) {
|
|
return minio.removeObject(STORAGE_BUCKET, objectName(a));
|
|
}
|
|
async function process2(a, history) {
|
|
try {
|
|
const results = await prisma.$transaction(async (prisma2) => {
|
|
const meme = memes.fromAttachment(a);
|
|
const dbMeme = await prisma2.meme.upsert({ create: meme, update: meme, where: { id: meme.id } });
|
|
const dbA = await prisma2.attachment.upsert({ create: a, update: a, where: { id: a.id } });
|
|
log.debug({ msg: "pending addition to db", dbMeme });
|
|
const upload = await storeAttachment(a);
|
|
log.debug({ msg: "successfully uploaded", upload });
|
|
await memes.persist(dbMeme);
|
|
log.info(dbMeme, "new meme");
|
|
return { meme: dbMeme, upload, attachment: dbA };
|
|
}, { timeout: 6e4 });
|
|
} catch (e) {
|
|
if (e.meta?.modelName !== "Meme" || e.meta?.target?.[0] !== "attachmentId") {
|
|
log.error(e, "failed to process attachment");
|
|
log.error(_.omit(a, ["vision"]), "failed to process attachment");
|
|
}
|
|
await deleteAttachment(a);
|
|
}
|
|
const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id } });
|
|
log.debug({ newHistory }, "updated channel history");
|
|
return true;
|
|
}
|
|
await ensureStorageBucketExists();
|
|
return {
|
|
process: process2
|
|
};
|
|
}
|
|
|
|
// src/scraper.ts
|
|
function convertSnowflakeToDate(snowflake) {
|
|
const DISCORD_EPOCH = 14200704e5;
|
|
const milliseconds = global.BigInt(snowflake) >> 22n;
|
|
return new Date(Number(milliseconds) + DISCORD_EPOCH);
|
|
}
|
|
async function scraper(discordAuth, vision, minio, prisma, baseLog, attachments) {
|
|
const headers = {
|
|
"Authorization": discordAuth
|
|
};
|
|
async function fetchChannelMessages(channelId) {
|
|
const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`;
|
|
const res = await axios2.get(url, { headers });
|
|
return res.data;
|
|
}
|
|
async function fetchChannelMessagesAfter(channelId, lastMessageId) {
|
|
const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`;
|
|
const res = await axios2.get(url, { headers });
|
|
return res.data;
|
|
}
|
|
async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
|
|
const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`;
|
|
const res = await axios2.get(url, { headers });
|
|
return res.data;
|
|
}
|
|
async function fetchGuildDetails(guildId) {
|
|
const url = `https://discord.com/api/v9/guilds/${guildId}`;
|
|
const res = await axios2.get(url, { headers });
|
|
return _2.omit(res.data, ["emojis", "stickers", "roles", "features"]);
|
|
}
|
|
async function fetchChannel(channelId) {
|
|
const url = `https://discord.com/api/v9/channels/${channelId}`;
|
|
const res = await axios2.get(url, { headers });
|
|
return res.data;
|
|
}
|
|
async function detectText(attachment) {
|
|
if (!IMAGE_TYPES.includes(attachment.contentType))
|
|
return {};
|
|
const annotateRequest = {
|
|
image: {
|
|
source: {
|
|
imageUri: attachment.url
|
|
}
|
|
},
|
|
features: [{ type: "TEXT_DETECTION" }]
|
|
};
|
|
const res = await vision.annotateImage(annotateRequest);
|
|
return res[0] || {};
|
|
}
|
|
async function fetchMissingVision() {
|
|
const attachments2 = await prisma.attachment.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } } });
|
|
baseLog.info({ attachments: attachments2.length }, "missing vision attachments");
|
|
}
|
|
async function processChannel(channelId) {
|
|
const channel = await fetchChannel(channelId);
|
|
const guild = await fetchGuildDetails(channel.guild_id);
|
|
const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId });
|
|
async function filterAttachments(messages) {
|
|
const acceptedContentTypes = [
|
|
...IMAGE_TYPES
|
|
// ...VIDEO_TYPES,
|
|
];
|
|
const validAttachments = messages.filter((m) => m.attachments).flatMap((m) => m.attachments.map((a) => [a, m])).filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
|
|
log.debug({ validAttachments: validAttachments.length }, "attachments found in messages");
|
|
return Promise.all(validAttachments.map(async ([a, m]) => {
|
|
const attachment = {
|
|
id: BigInt(a.id),
|
|
msgId: BigInt(m.id),
|
|
channelId: BigInt(m.channel_id),
|
|
url: a.url,
|
|
contentType: a.content_type,
|
|
// create this ahead of time to associate with our copy of the media
|
|
memeId: crypto.randomUUID(),
|
|
vision: {},
|
|
// quick hack so that detectText can receive a full attachment
|
|
guild,
|
|
channel,
|
|
message: m,
|
|
postedAt: convertSnowflakeToDate(a.id),
|
|
// ts asks for these to be included but they don't need to be set
|
|
createdAt: void 0,
|
|
updatedAt: void 0
|
|
};
|
|
return { ...attachment, vision: await detectText(attachment) };
|
|
}));
|
|
}
|
|
async function* fetchNew(channelId2, log2, latestMessageId) {
|
|
const messages = (await fetchChannelMessagesAfter(channelId2, latestMessageId)).reverse();
|
|
if (messages.length === 0) {
|
|
log2.info("reached latest message");
|
|
}
|
|
log2.debug({ messages: messages.length, latestMessageId }, "fetched new messages");
|
|
yield* await filterAttachments(messages);
|
|
}
|
|
async function* fetchOld(channelId2, log2, earliestMessageId) {
|
|
const messages = await fetchChannelMessagesBefore(channelId2, earliestMessageId);
|
|
if (messages.length === 0) {
|
|
log2.info("reached beginning of history");
|
|
}
|
|
log2.debug({ messages: messages.length, earliestMessageId }, "fetched older messages");
|
|
yield* await filterAttachments(messages);
|
|
}
|
|
const ensureHistory = async () => {
|
|
const stored = await prisma.channelHistory.findFirst({ where: { id: channelId } });
|
|
if (stored)
|
|
return stored;
|
|
const lastMessageId = channel.last_message_id;
|
|
return await prisma.channelHistory.create({
|
|
data: { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
|
|
});
|
|
};
|
|
const history = await ensureHistory();
|
|
async function processLoop() {
|
|
const history2 = await ensureHistory();
|
|
const newAttachments = fetchNew(channelId, log, history2.latestMessageId);
|
|
const oldAttachments = fetchOld(channelId, log, history2.earliestMessageId);
|
|
let oldCount = 0;
|
|
let newCount = 0;
|
|
for await (const attachment of newAttachments) {
|
|
log.debug({ msg: "scraped new attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
|
const newHistory = { id: history2.id, latestMessageId: attachment.msgId };
|
|
await attachments.process(attachment, newHistory);
|
|
oldCount++;
|
|
}
|
|
for await (const attachment of oldAttachments) {
|
|
log.debug({ msg: "scraped old attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
|
const newHistory = { id: history2.id, earliestMessageId: attachment.msgId };
|
|
await attachments.process(attachment, newHistory);
|
|
newCount++;
|
|
}
|
|
log.info({ oldCount, newCount }, "processed attachments");
|
|
const shouldWait = newCount === 0 && oldCount === 0;
|
|
const PROCESS_DELAY_MS = 1e3 * 60;
|
|
if (shouldWait) {
|
|
log.info({ PROCESS_DELAY_MS }, "waiting for new messages");
|
|
await setTimeout(PROCESS_DELAY_MS);
|
|
}
|
|
return processLoop();
|
|
}
|
|
await processLoop();
|
|
}
|
|
return {
|
|
processChannel
|
|
};
|
|
}
|
|
|
|
// src/memes.ts
|
|
import _3 from "lodash";
|
|
var STORAGE_BUCKET2 = process.env.STORAGE_BUCKET;
|
|
function memeUrl(a) {
|
|
const STORAGE_URL = process.env.STORAGE_URL;
|
|
const u = new URL(`${STORAGE_BUCKET2}/${objectName(a)}`, `https://${STORAGE_URL}`);
|
|
return u.href;
|
|
}
|
|
async function memeService(baseLog, prisma, meili) {
|
|
const log = baseLog.child({ service: "meme" });
|
|
function fromAttachment(a) {
|
|
const text = a.vision.fullTextAnnotation?.text || "";
|
|
const tags = [
|
|
a.guild.name,
|
|
a.channel.name,
|
|
a.message.author.username
|
|
];
|
|
const m = {
|
|
id: a.memeId,
|
|
url: memeUrl(a),
|
|
text,
|
|
tags,
|
|
attachmentId: a.id,
|
|
postedAt: a.postedAt,
|
|
createdAt: void 0,
|
|
updatedAt: void 0
|
|
};
|
|
return m;
|
|
}
|
|
async function ensureMeiliIndex() {
|
|
const task = await meili.createIndex("memes", { primaryKey: "id" });
|
|
const result = await meili.waitForTask(task.taskUid);
|
|
log.debug(result);
|
|
const index = await meili.index("memes");
|
|
await index.updateSortableAttributes(["postedAt"]);
|
|
return index;
|
|
}
|
|
async function resetMeili() {
|
|
const delTask = await meili.deleteIndex("memes");
|
|
const delResult = await meili.waitForTask(delTask.taskUid);
|
|
const index = await ensureMeiliIndex();
|
|
const memes = (await prisma.meme.findMany()).map((m) => ({ ...m, postedAt: m.postedAt.valueOf() }));
|
|
for await (const batch of _3.chunk(memes, 1e3)) {
|
|
const task = await index.updateDocuments(batch);
|
|
const res = await meili.waitForTask(task.taskUid);
|
|
log.info({ res }, "added meme batch");
|
|
}
|
|
}
|
|
async function persist(m) {
|
|
const index = meili.index("memes");
|
|
const doc = { ...m, postedAt: m.postedAt.valueOf() };
|
|
return index.updateDocuments([doc]);
|
|
}
|
|
await ensureMeiliIndex();
|
|
return {
|
|
ensureMeiliIndex,
|
|
resetMeili,
|
|
fromAttachment,
|
|
persist
|
|
};
|
|
}
|
|
|
|
// index.ts
|
|
BigInt.prototype.toJSON = function bigIntStringify() {
|
|
return this.toString();
|
|
};
|
|
var exec = promisify(execCb);
|
|
async function main() {
|
|
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
|
|
const log = pino({ base: {} });
|
|
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS });
|
|
const prisma = new PrismaClient();
|
|
const minio = new Minio.Client({
|
|
endPoint: process.env.STORAGE_URL,
|
|
accessKey: process.env.STORAGE_ACCESS_KEY,
|
|
secretKey: process.env.STORAGE_SECRET_KEY
|
|
});
|
|
const meili = new MeiliSearch({
|
|
host: process.env.MEILI_URL,
|
|
apiKey: process.env.MEILI_MASTER_KEY
|
|
});
|
|
const memes = await memeService(log, prisma, meili);
|
|
const attachments = await attachmentService(log, minio, prisma, memes);
|
|
const scraper2 = await scraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments);
|
|
const CHANNEL_IDS = [
|
|
367936426962190336n,
|
|
// TPM shitposts
|
|
367936467592413194n,
|
|
// tpm screenshots
|
|
1028511952277864501n,
|
|
// tpm sludge
|
|
725835129326665768n,
|
|
// machinegirl memes
|
|
1095060717117374687n
|
|
// aella memes
|
|
];
|
|
async function migrateDb() {
|
|
const output = await exec("prisma migrate deploy");
|
|
log.info(output);
|
|
}
|
|
await migrateDb();
|
|
await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
|
|
}
|
|
main().then(() => true).catch((e) => console.error(e));
|