combination WIP, add client
This commit is contained in:
Nathan Rashleigh committed
1 parent
3c5e7d9aee
commit
4097b05cf9
44 files changed
+22234
-5001
No files matched your search
@@ -0,0 +1,336 @@
|
||||
// index.ts
|
||||
import "dotenv/config";
|
||||
import pino from "pino";
|
||||
import * as Minio from "minio";
|
||||
import Vision from "@google-cloud/vision";
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { MeiliSearch } from "meilisearch";
|
||||
|
||||
// src/scraper.ts
|
||||
import { setTimeout } from "timers/promises";
|
||||
import axios2 from "axios";
|
||||
import _2 from "lodash";
|
||||
|
||||
// src/attachments.ts
|
||||
import sharp from "sharp";
|
||||
import axios from "axios";
|
||||
import mime from "mime-types";
|
||||
import _ from "lodash";
|
||||
var STORAGE_BUCKET = process.env.STORAGE_BUCKET;
|
||||
var IMAGE_TYPES = [
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/webp",
|
||||
"image/gif"
|
||||
];
|
||||
function isImage(a) {
|
||||
return IMAGE_TYPES.includes(a.contentType);
|
||||
}
|
||||
function objectName(a) {
|
||||
if (isImage(a))
|
||||
return `${a.memeId}.webp`;
|
||||
return `${a.memeId}.${mime.extension(a.contentType)}`;
|
||||
}
|
||||
async function attachmentService(log, minio, prisma, memes) {
|
||||
async function ensureStorageBucketExists() {
|
||||
const exists = await minio.bucketExists(STORAGE_BUCKET);
|
||||
if (exists)
|
||||
return true;
|
||||
await minio.makeBucket(STORAGE_BUCKET);
|
||||
}
|
||||
async function storeAttachment(a) {
|
||||
const download = await axios.get(a.url, { responseType: "stream" });
|
||||
const stream = () => {
|
||||
if (isImage(a)) {
|
||||
const converter = sharp().webp();
|
||||
download.data.pipe(converter);
|
||||
return converter;
|
||||
}
|
||||
return download.data;
|
||||
};
|
||||
return minio.putObject(
|
||||
STORAGE_BUCKET,
|
||||
objectName(a),
|
||||
stream(),
|
||||
null,
|
||||
// hardcode this for now, we upload everything as webp
|
||||
{ "Content-Type": "image/webp" }
|
||||
);
|
||||
}
|
||||
async function deleteAttachment(a) {
|
||||
return minio.removeObject(STORAGE_BUCKET, objectName(a));
|
||||
}
|
||||
async function process2(a, history) {
|
||||
try {
|
||||
const results = await prisma.$transaction(async (prisma2) => {
|
||||
const meme = memes.fromAttachment(a);
|
||||
const dbMeme = await prisma2.meme.upsert({ create: meme, update: meme, where: { id: meme.id } });
|
||||
const dbA = await prisma2.attachment.upsert({ create: a, update: a, where: { id: a.id } });
|
||||
log.debug({ msg: "pending addition to db", dbMeme });
|
||||
const upload = await storeAttachment(a);
|
||||
log.debug({ msg: "successfully uploaded", upload });
|
||||
await memes.persist(dbMeme);
|
||||
log.info(dbMeme, "new meme");
|
||||
return { meme: dbMeme, upload, attachment: dbA };
|
||||
}, { timeout: 6e4 });
|
||||
} catch (e) {
|
||||
if (e.meta?.modelName !== "Meme" || e.meta?.target?.[0] !== "attachmentId") {
|
||||
log.error(e, "failed to process attachment");
|
||||
log.error(_.omit(a, ["vision"]), "failed to process attachment");
|
||||
}
|
||||
await deleteAttachment(a);
|
||||
}
|
||||
const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id } });
|
||||
log.debug({ newHistory }, "updated channel history");
|
||||
return true;
|
||||
}
|
||||
await ensureStorageBucketExists();
|
||||
return {
|
||||
process: process2
|
||||
};
|
||||
}
|
||||
|
||||
// src/scraper.ts
|
||||
function convertSnowflakeToDate(snowflake) {
|
||||
const DISCORD_EPOCH = 14200704e5;
|
||||
const milliseconds = global.BigInt(snowflake) >> 22n;
|
||||
return new Date(Number(milliseconds) + DISCORD_EPOCH);
|
||||
}
|
||||
async function scraper(discordAuth, vision, minio, prisma, baseLog, attachments) {
|
||||
const headers = {
|
||||
"Authorization": discordAuth
|
||||
};
|
||||
async function fetchChannelMessages(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchChannelMessagesAfter(channelId, lastMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchGuildDetails(guildId) {
|
||||
const url = `https://discord.com/api/v9/guilds/${guildId}`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return _2.omit(res.data, ["emojis", "stickers", "roles", "features"]);
|
||||
}
|
||||
async function fetchChannel(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function detectText(attachment) {
|
||||
if (!IMAGE_TYPES.includes(attachment.contentType))
|
||||
return {};
|
||||
const annotateRequest = {
|
||||
image: {
|
||||
source: {
|
||||
imageUri: attachment.url
|
||||
}
|
||||
},
|
||||
features: [{ type: "TEXT_DETECTION" }]
|
||||
};
|
||||
const res = await vision.annotateImage(annotateRequest);
|
||||
return res[0] || {};
|
||||
}
|
||||
async function fetchMissingVision() {
|
||||
const attachments2 = await prisma.attachment.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } } });
|
||||
baseLog.info({ attachments: attachments2.length }, "missing vision attachments");
|
||||
}
|
||||
async function processChannel(channelId) {
|
||||
const channel = await fetchChannel(channelId);
|
||||
const guild = await fetchGuildDetails(channel.guild_id);
|
||||
const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId });
|
||||
async function filterAttachments(messages) {
|
||||
const acceptedContentTypes = [
|
||||
...IMAGE_TYPES
|
||||
// ...VIDEO_TYPES,
|
||||
];
|
||||
const validAttachments = messages.filter((m) => m.attachments).flatMap((m) => m.attachments.map((a) => [a, m])).filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
|
||||
log.debug({ validAttachments: validAttachments.length }, "attachments found in messages");
|
||||
return Promise.all(validAttachments.map(async ([a, m]) => {
|
||||
const attachment = {
|
||||
id: BigInt(a.id),
|
||||
msgId: BigInt(m.id),
|
||||
channelId: BigInt(m.channel_id),
|
||||
url: a.url,
|
||||
contentType: a.content_type,
|
||||
// create this ahead of time to associate with our copy of the media
|
||||
memeId: crypto.randomUUID(),
|
||||
vision: {},
|
||||
// quick hack so that detectText can receive a full attachment
|
||||
guild,
|
||||
channel,
|
||||
message: m,
|
||||
postedAt: convertSnowflakeToDate(a.id),
|
||||
// ts asks for these to be included but they don't need to be set
|
||||
createdAt: void 0,
|
||||
updatedAt: void 0
|
||||
};
|
||||
return { ...attachment, vision: await detectText(attachment) };
|
||||
}));
|
||||
}
|
||||
async function* fetchNew(channelId2, log2, latestMessageId) {
|
||||
const messages = (await fetchChannelMessagesAfter(channelId2, latestMessageId)).reverse();
|
||||
if (messages.length === 0) {
|
||||
log2.info("reached latest message");
|
||||
}
|
||||
log2.debug({ messages: messages.length, latestMessageId }, "fetched new messages");
|
||||
yield* await filterAttachments(messages);
|
||||
}
|
||||
async function* fetchOld(channelId2, log2, earliestMessageId) {
|
||||
const messages = await fetchChannelMessagesBefore(channelId2, earliestMessageId);
|
||||
if (messages.length === 0) {
|
||||
log2.info("reached beginning of history");
|
||||
}
|
||||
log2.debug({ messages: messages.length, earliestMessageId }, "fetched older messages");
|
||||
yield* await filterAttachments(messages);
|
||||
}
|
||||
const ensureHistory = async () => {
|
||||
const stored = await prisma.channelHistory.findFirst({ where: { id: channelId } });
|
||||
if (stored)
|
||||
return stored;
|
||||
const lastMessageId = channel.last_message_id;
|
||||
return await prisma.channelHistory.create({
|
||||
data: { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
|
||||
});
|
||||
};
|
||||
const history = await ensureHistory();
|
||||
async function processLoop() {
|
||||
const history2 = await ensureHistory();
|
||||
const newAttachments = fetchNew(channelId, log, history2.latestMessageId);
|
||||
const oldAttachments = fetchOld(channelId, log, history2.earliestMessageId);
|
||||
let oldCount = 0;
|
||||
let newCount = 0;
|
||||
for await (const attachment of newAttachments) {
|
||||
log.debug({ msg: "scraped new attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
||||
const newHistory = { id: history2.id, latestMessageId: attachment.msgId };
|
||||
await attachments.process(attachment, newHistory);
|
||||
oldCount++;
|
||||
}
|
||||
for await (const attachment of oldAttachments) {
|
||||
log.debug({ msg: "scraped old attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
||||
const newHistory = { id: history2.id, earliestMessageId: attachment.msgId };
|
||||
await attachments.process(attachment, newHistory);
|
||||
newCount++;
|
||||
}
|
||||
log.info({ oldCount, newCount }, "processed attachments");
|
||||
const shouldWait = newCount === 0 && oldCount === 0;
|
||||
const PROCESS_DELAY_MS = 1e3 * 60;
|
||||
if (shouldWait) {
|
||||
log.info({ PROCESS_DELAY_MS }, "waiting for new messages");
|
||||
await setTimeout(PROCESS_DELAY_MS);
|
||||
}
|
||||
return processLoop();
|
||||
}
|
||||
await processLoop();
|
||||
}
|
||||
return {
|
||||
processChannel
|
||||
};
|
||||
}
|
||||
|
||||
// src/memes.ts
|
||||
import _3 from "lodash";
|
||||
var STORAGE_BUCKET2 = process.env.STORAGE_BUCKET;
|
||||
function memeUrl(a) {
|
||||
const STORAGE_URL = process.env.STORAGE_URL;
|
||||
const u = new URL(`${STORAGE_BUCKET2}/${objectName(a)}`, `https://${STORAGE_URL}`);
|
||||
return u.href;
|
||||
}
|
||||
async function memeService(baseLog, prisma, meili) {
|
||||
const log = baseLog.child({ service: "meme" });
|
||||
function fromAttachment(a) {
|
||||
const text = a.vision.fullTextAnnotation?.text || "";
|
||||
const tags = [
|
||||
a.guild.name,
|
||||
a.channel.name,
|
||||
a.message.author.username
|
||||
];
|
||||
const m = {
|
||||
id: a.memeId,
|
||||
url: memeUrl(a),
|
||||
text,
|
||||
tags,
|
||||
attachmentId: a.id,
|
||||
postedAt: a.postedAt,
|
||||
createdAt: void 0,
|
||||
updatedAt: void 0
|
||||
};
|
||||
return m;
|
||||
}
|
||||
async function ensureMeiliIndex() {
|
||||
const task = await meili.createIndex("memes", { primaryKey: "id" });
|
||||
const result = await meili.waitForTask(task.taskUid);
|
||||
log.debug(result);
|
||||
const index = await meili.index("memes");
|
||||
await index.updateSortableAttributes(["postedAt"]);
|
||||
return index;
|
||||
}
|
||||
async function resetMeili() {
|
||||
const delTask = await meili.deleteIndex("memes");
|
||||
const delResult = await meili.waitForTask(delTask.taskUid);
|
||||
const index = await ensureMeiliIndex();
|
||||
const memes = (await prisma.meme.findMany()).map((m) => ({ ...m, postedAt: m.postedAt.valueOf() }));
|
||||
for await (const batch of _3.chunk(memes, 1e3)) {
|
||||
const task = await index.updateDocuments(batch);
|
||||
const res = await meili.waitForTask(task.taskUid);
|
||||
log.info({ res }, "added meme batch");
|
||||
}
|
||||
}
|
||||
async function persist(m) {
|
||||
const index = meili.index("memes");
|
||||
const doc = { ...m, postedAt: m.postedAt.valueOf() };
|
||||
return index.updateDocuments([doc]);
|
||||
}
|
||||
await ensureMeiliIndex();
|
||||
return {
|
||||
ensureMeiliIndex,
|
||||
resetMeili,
|
||||
fromAttachment,
|
||||
persist
|
||||
};
|
||||
}
|
||||
|
||||
// index.ts
|
||||
BigInt.prototype.toJSON = function bigIntStringify() {
|
||||
return this.toString();
|
||||
};
|
||||
async function main() {
|
||||
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
|
||||
const log = pino({ base: {} });
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS });
|
||||
const prisma = new PrismaClient();
|
||||
const minio = new Minio.Client({
|
||||
endPoint: process.env.STORAGE_URL,
|
||||
accessKey: process.env.STORAGE_ACCESS_KEY,
|
||||
secretKey: process.env.STORAGE_SECRET_KEY
|
||||
});
|
||||
const meili = new MeiliSearch({
|
||||
host: process.env.MEILI_URL,
|
||||
apiKey: process.env.MEILI_MASTER_KEY
|
||||
});
|
||||
const memes = await memeService(log, prisma, meili);
|
||||
const attachments = await attachmentService(log, minio, prisma, memes);
|
||||
const scraper2 = await scraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments);
|
||||
const CHANNEL_IDS = [
|
||||
367936426962190336n,
|
||||
// TPM shitposts
|
||||
367936467592413194n,
|
||||
// tpm screenshots
|
||||
1028511952277864501n,
|
||||
// tpm sludge
|
||||
725835129326665768n,
|
||||
// machinegirl memes
|
||||
1095060717117374687n
|
||||
// aella memes
|
||||
];
|
||||
await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
|
||||
}
|
||||
main().then(() => true).catch((e) => console.error(e));
|
||||
Reference in new issue
Block a user