combination WIP, add client
This commit is contained in:
Nathan Rashleigh committed
1 parent
3c5e7d9aee
commit
4097b05cf9
44 files changed
+22234
-5001
No files matched your search
@@ -0,0 +1,21 @@
|
||||
FROM node:20 as builder
|
||||
WORKDIR /app
|
||||
|
||||
COPY . ./
|
||||
RUN npm i
|
||||
RUN npm run build
|
||||
|
||||
# execution
|
||||
FROM node:20
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/build /app
|
||||
|
||||
# packages are not bundled in the build because sharp uses native modules
|
||||
COPY package.json /app/package.json
|
||||
RUN npm i --include prod
|
||||
|
||||
# prisma setup
|
||||
COPY ./prisma /app/prisma
|
||||
RUN npx prisma generate
|
||||
|
||||
CMD node index.js
|
||||
@@ -0,0 +1,8 @@
|
||||
# memestream
|
||||
|
||||
## TODO
|
||||
|
||||
[ ] Backfill last N images from a channel
|
||||
[ ] Set oldest_img_id for channel
|
||||
[ ]
|
||||
[ ] change downloader to url type
|
||||
@@ -0,0 +1,336 @@
|
||||
// index.ts
|
||||
import "dotenv/config";
|
||||
import pino from "pino";
|
||||
import * as Minio from "minio";
|
||||
import Vision from "@google-cloud/vision";
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { MeiliSearch } from "meilisearch";
|
||||
|
||||
// src/scraper.ts
|
||||
import { setTimeout } from "timers/promises";
|
||||
import axios2 from "axios";
|
||||
import _2 from "lodash";
|
||||
|
||||
// src/attachments.ts
|
||||
import sharp from "sharp";
|
||||
import axios from "axios";
|
||||
import mime from "mime-types";
|
||||
import _ from "lodash";
|
||||
var STORAGE_BUCKET = process.env.STORAGE_BUCKET;
|
||||
var IMAGE_TYPES = [
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/webp",
|
||||
"image/gif"
|
||||
];
|
||||
function isImage(a) {
|
||||
return IMAGE_TYPES.includes(a.contentType);
|
||||
}
|
||||
function objectName(a) {
|
||||
if (isImage(a))
|
||||
return `${a.memeId}.webp`;
|
||||
return `${a.memeId}.${mime.extension(a.contentType)}`;
|
||||
}
|
||||
async function attachmentService(log, minio, prisma, memes) {
|
||||
async function ensureStorageBucketExists() {
|
||||
const exists = await minio.bucketExists(STORAGE_BUCKET);
|
||||
if (exists)
|
||||
return true;
|
||||
await minio.makeBucket(STORAGE_BUCKET);
|
||||
}
|
||||
async function storeAttachment(a) {
|
||||
const download = await axios.get(a.url, { responseType: "stream" });
|
||||
const stream = () => {
|
||||
if (isImage(a)) {
|
||||
const converter = sharp().webp();
|
||||
download.data.pipe(converter);
|
||||
return converter;
|
||||
}
|
||||
return download.data;
|
||||
};
|
||||
return minio.putObject(
|
||||
STORAGE_BUCKET,
|
||||
objectName(a),
|
||||
stream(),
|
||||
null,
|
||||
// hardcode this for now, we upload everything as webp
|
||||
{ "Content-Type": "image/webp" }
|
||||
);
|
||||
}
|
||||
async function deleteAttachment(a) {
|
||||
return minio.removeObject(STORAGE_BUCKET, objectName(a));
|
||||
}
|
||||
async function process2(a, history) {
|
||||
try {
|
||||
const results = await prisma.$transaction(async (prisma2) => {
|
||||
const meme = memes.fromAttachment(a);
|
||||
const dbMeme = await prisma2.meme.upsert({ create: meme, update: meme, where: { id: meme.id } });
|
||||
const dbA = await prisma2.attachment.upsert({ create: a, update: a, where: { id: a.id } });
|
||||
log.debug({ msg: "pending addition to db", dbMeme });
|
||||
const upload = await storeAttachment(a);
|
||||
log.debug({ msg: "successfully uploaded", upload });
|
||||
await memes.persist(dbMeme);
|
||||
log.info(dbMeme, "new meme");
|
||||
return { meme: dbMeme, upload, attachment: dbA };
|
||||
}, { timeout: 6e4 });
|
||||
} catch (e) {
|
||||
if (e.meta?.modelName !== "Meme" || e.meta?.target?.[0] !== "attachmentId") {
|
||||
log.error(e, "failed to process attachment");
|
||||
log.error(_.omit(a, ["vision"]), "failed to process attachment");
|
||||
}
|
||||
await deleteAttachment(a);
|
||||
}
|
||||
const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id } });
|
||||
log.debug({ newHistory }, "updated channel history");
|
||||
return true;
|
||||
}
|
||||
await ensureStorageBucketExists();
|
||||
return {
|
||||
process: process2
|
||||
};
|
||||
}
|
||||
|
||||
// src/scraper.ts
|
||||
function convertSnowflakeToDate(snowflake) {
|
||||
const DISCORD_EPOCH = 14200704e5;
|
||||
const milliseconds = global.BigInt(snowflake) >> 22n;
|
||||
return new Date(Number(milliseconds) + DISCORD_EPOCH);
|
||||
}
|
||||
async function scraper(discordAuth, vision, minio, prisma, baseLog, attachments) {
|
||||
const headers = {
|
||||
"Authorization": discordAuth
|
||||
};
|
||||
async function fetchChannelMessages(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchChannelMessagesAfter(channelId, lastMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function fetchGuildDetails(guildId) {
|
||||
const url = `https://discord.com/api/v9/guilds/${guildId}`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return _2.omit(res.data, ["emojis", "stickers", "roles", "features"]);
|
||||
}
|
||||
async function fetchChannel(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}`;
|
||||
const res = await axios2.get(url, { headers });
|
||||
return res.data;
|
||||
}
|
||||
async function detectText(attachment) {
|
||||
if (!IMAGE_TYPES.includes(attachment.contentType))
|
||||
return {};
|
||||
const annotateRequest = {
|
||||
image: {
|
||||
source: {
|
||||
imageUri: attachment.url
|
||||
}
|
||||
},
|
||||
features: [{ type: "TEXT_DETECTION" }]
|
||||
};
|
||||
const res = await vision.annotateImage(annotateRequest);
|
||||
return res[0] || {};
|
||||
}
|
||||
async function fetchMissingVision() {
|
||||
const attachments2 = await prisma.attachment.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } } });
|
||||
baseLog.info({ attachments: attachments2.length }, "missing vision attachments");
|
||||
}
|
||||
async function processChannel(channelId) {
|
||||
const channel = await fetchChannel(channelId);
|
||||
const guild = await fetchGuildDetails(channel.guild_id);
|
||||
const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId });
|
||||
async function filterAttachments(messages) {
|
||||
const acceptedContentTypes = [
|
||||
...IMAGE_TYPES
|
||||
// ...VIDEO_TYPES,
|
||||
];
|
||||
const validAttachments = messages.filter((m) => m.attachments).flatMap((m) => m.attachments.map((a) => [a, m])).filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
|
||||
log.debug({ validAttachments: validAttachments.length }, "attachments found in messages");
|
||||
return Promise.all(validAttachments.map(async ([a, m]) => {
|
||||
const attachment = {
|
||||
id: BigInt(a.id),
|
||||
msgId: BigInt(m.id),
|
||||
channelId: BigInt(m.channel_id),
|
||||
url: a.url,
|
||||
contentType: a.content_type,
|
||||
// create this ahead of time to associate with our copy of the media
|
||||
memeId: crypto.randomUUID(),
|
||||
vision: {},
|
||||
// quick hack so that detectText can receive a full attachment
|
||||
guild,
|
||||
channel,
|
||||
message: m,
|
||||
postedAt: convertSnowflakeToDate(a.id),
|
||||
// ts asks for these to be included but they don't need to be set
|
||||
createdAt: void 0,
|
||||
updatedAt: void 0
|
||||
};
|
||||
return { ...attachment, vision: await detectText(attachment) };
|
||||
}));
|
||||
}
|
||||
async function* fetchNew(channelId2, log2, latestMessageId) {
|
||||
const messages = (await fetchChannelMessagesAfter(channelId2, latestMessageId)).reverse();
|
||||
if (messages.length === 0) {
|
||||
log2.info("reached latest message");
|
||||
}
|
||||
log2.debug({ messages: messages.length, latestMessageId }, "fetched new messages");
|
||||
yield* await filterAttachments(messages);
|
||||
}
|
||||
async function* fetchOld(channelId2, log2, earliestMessageId) {
|
||||
const messages = await fetchChannelMessagesBefore(channelId2, earliestMessageId);
|
||||
if (messages.length === 0) {
|
||||
log2.info("reached beginning of history");
|
||||
}
|
||||
log2.debug({ messages: messages.length, earliestMessageId }, "fetched older messages");
|
||||
yield* await filterAttachments(messages);
|
||||
}
|
||||
const ensureHistory = async () => {
|
||||
const stored = await prisma.channelHistory.findFirst({ where: { id: channelId } });
|
||||
if (stored)
|
||||
return stored;
|
||||
const lastMessageId = channel.last_message_id;
|
||||
return await prisma.channelHistory.create({
|
||||
data: { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
|
||||
});
|
||||
};
|
||||
const history = await ensureHistory();
|
||||
async function processLoop() {
|
||||
const history2 = await ensureHistory();
|
||||
const newAttachments = fetchNew(channelId, log, history2.latestMessageId);
|
||||
const oldAttachments = fetchOld(channelId, log, history2.earliestMessageId);
|
||||
let oldCount = 0;
|
||||
let newCount = 0;
|
||||
for await (const attachment of newAttachments) {
|
||||
log.debug({ msg: "scraped new attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
||||
const newHistory = { id: history2.id, latestMessageId: attachment.msgId };
|
||||
await attachments.process(attachment, newHistory);
|
||||
oldCount++;
|
||||
}
|
||||
for await (const attachment of oldAttachments) {
|
||||
log.debug({ msg: "scraped old attachment", attachment: _2.omit(attachment, ["guild", "message", "channel", "vision"]) });
|
||||
const newHistory = { id: history2.id, earliestMessageId: attachment.msgId };
|
||||
await attachments.process(attachment, newHistory);
|
||||
newCount++;
|
||||
}
|
||||
log.info({ oldCount, newCount }, "processed attachments");
|
||||
const shouldWait = newCount === 0 && oldCount === 0;
|
||||
const PROCESS_DELAY_MS = 1e3 * 60;
|
||||
if (shouldWait) {
|
||||
log.info({ PROCESS_DELAY_MS }, "waiting for new messages");
|
||||
await setTimeout(PROCESS_DELAY_MS);
|
||||
}
|
||||
return processLoop();
|
||||
}
|
||||
await processLoop();
|
||||
}
|
||||
return {
|
||||
processChannel
|
||||
};
|
||||
}
|
||||
|
||||
// src/memes.ts
|
||||
import _3 from "lodash";
|
||||
var STORAGE_BUCKET2 = process.env.STORAGE_BUCKET;
|
||||
function memeUrl(a) {
|
||||
const STORAGE_URL = process.env.STORAGE_URL;
|
||||
const u = new URL(`${STORAGE_BUCKET2}/${objectName(a)}`, `https://${STORAGE_URL}`);
|
||||
return u.href;
|
||||
}
|
||||
async function memeService(baseLog, prisma, meili) {
|
||||
const log = baseLog.child({ service: "meme" });
|
||||
function fromAttachment(a) {
|
||||
const text = a.vision.fullTextAnnotation?.text || "";
|
||||
const tags = [
|
||||
a.guild.name,
|
||||
a.channel.name,
|
||||
a.message.author.username
|
||||
];
|
||||
const m = {
|
||||
id: a.memeId,
|
||||
url: memeUrl(a),
|
||||
text,
|
||||
tags,
|
||||
attachmentId: a.id,
|
||||
postedAt: a.postedAt,
|
||||
createdAt: void 0,
|
||||
updatedAt: void 0
|
||||
};
|
||||
return m;
|
||||
}
|
||||
async function ensureMeiliIndex() {
|
||||
const task = await meili.createIndex("memes", { primaryKey: "id" });
|
||||
const result = await meili.waitForTask(task.taskUid);
|
||||
log.debug(result);
|
||||
const index = await meili.index("memes");
|
||||
await index.updateSortableAttributes(["postedAt"]);
|
||||
return index;
|
||||
}
|
||||
async function resetMeili() {
|
||||
const delTask = await meili.deleteIndex("memes");
|
||||
const delResult = await meili.waitForTask(delTask.taskUid);
|
||||
const index = await ensureMeiliIndex();
|
||||
const memes = (await prisma.meme.findMany()).map((m) => ({ ...m, postedAt: m.postedAt.valueOf() }));
|
||||
for await (const batch of _3.chunk(memes, 1e3)) {
|
||||
const task = await index.updateDocuments(batch);
|
||||
const res = await meili.waitForTask(task.taskUid);
|
||||
log.info({ res }, "added meme batch");
|
||||
}
|
||||
}
|
||||
async function persist(m) {
|
||||
const index = meili.index("memes");
|
||||
const doc = { ...m, postedAt: m.postedAt.valueOf() };
|
||||
return index.updateDocuments([doc]);
|
||||
}
|
||||
await ensureMeiliIndex();
|
||||
return {
|
||||
ensureMeiliIndex,
|
||||
resetMeili,
|
||||
fromAttachment,
|
||||
persist
|
||||
};
|
||||
}
|
||||
|
||||
// index.ts
|
||||
BigInt.prototype.toJSON = function bigIntStringify() {
|
||||
return this.toString();
|
||||
};
|
||||
async function main() {
|
||||
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
|
||||
const log = pino({ base: {} });
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS });
|
||||
const prisma = new PrismaClient();
|
||||
const minio = new Minio.Client({
|
||||
endPoint: process.env.STORAGE_URL,
|
||||
accessKey: process.env.STORAGE_ACCESS_KEY,
|
||||
secretKey: process.env.STORAGE_SECRET_KEY
|
||||
});
|
||||
const meili = new MeiliSearch({
|
||||
host: process.env.MEILI_URL,
|
||||
apiKey: process.env.MEILI_MASTER_KEY
|
||||
});
|
||||
const memes = await memeService(log, prisma, meili);
|
||||
const attachments = await attachmentService(log, minio, prisma, memes);
|
||||
const scraper2 = await scraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments);
|
||||
const CHANNEL_IDS = [
|
||||
367936426962190336n,
|
||||
// TPM shitposts
|
||||
367936467592413194n,
|
||||
// tpm screenshots
|
||||
1028511952277864501n,
|
||||
// tpm sludge
|
||||
725835129326665768n,
|
||||
// machinegirl memes
|
||||
1095060717117374687n
|
||||
// aella memes
|
||||
];
|
||||
await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
|
||||
}
|
||||
main().then(() => true).catch((e) => console.error(e));
|
||||
@@ -0,0 +1,68 @@
|
||||
import "dotenv/config"
|
||||
|
||||
import fs from 'fs/promises'
|
||||
import { exec as execCb } from 'child_process'
|
||||
import { promisify } from 'util'
|
||||
|
||||
import pino from 'pino'
|
||||
|
||||
import * as Minio from 'minio'
|
||||
import Vision from '@google-cloud/vision'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { MeiliSearch } from 'meilisearch'
|
||||
|
||||
import discordScraper from './src/scraper'
|
||||
import attachmentService from "./src/attachments"
|
||||
import memeService from "./src/memes"
|
||||
|
||||
// @ts-ignore - ts don't know wtf it talking about
|
||||
BigInt.prototype.toJSON = function bigIntStringify() {
|
||||
return this.toString();
|
||||
};
|
||||
|
||||
const exec = promisify(execCb)
|
||||
|
||||
async function main() {
|
||||
const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default
|
||||
|
||||
const log = pino({ base: {} });
|
||||
|
||||
const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
|
||||
const prisma = new PrismaClient()
|
||||
const minio = new Minio.Client({
|
||||
endPoint: process.env.STORAGE_URL,
|
||||
accessKey: process.env.STORAGE_ACCESS_KEY,
|
||||
secretKey: process.env.STORAGE_SECRET_KEY,
|
||||
})
|
||||
const meili = new MeiliSearch({
|
||||
host: process.env.MEILI_URL,
|
||||
apiKey: process.env.MEILI_MASTER_KEY,
|
||||
})
|
||||
|
||||
const memes = await memeService(log, prisma, meili)
|
||||
const attachments = await attachmentService(log, minio, prisma, memes)
|
||||
const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
|
||||
|
||||
const CHANNEL_IDS = [
|
||||
367936426962190336n, // TPM shitposts
|
||||
367936467592413194n, // tpm screenshots
|
||||
1028511952277864501n, // tpm sludge
|
||||
725835129326665768n, // machinegirl memes
|
||||
1095060717117374687n, // aella memes
|
||||
]
|
||||
|
||||
async function migrateDb() {
|
||||
const output = await exec('prisma migrate deploy')
|
||||
log.info(output)
|
||||
}
|
||||
|
||||
await migrateDb()
|
||||
|
||||
await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
|
||||
}
|
||||
|
||||
main()
|
||||
.then(() => true)
|
||||
.catch(e => console.error(e))
|
||||
|
||||
|
||||
Generated
+2360
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "memestream",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "esrun index.ts | pino-pretty",
|
||||
"start:bundle": "node index.js",
|
||||
"dev": "esrun --watch index.ts | pino-pretty",
|
||||
"build": "esbuild index.ts --bundle --format=esm --outdir=build --platform=node --target=node22.1 --packages=external",
|
||||
"migrate:make": "prisma migrate dev --name",
|
||||
"test": "esrun src/scraper.test.ts | pino-pretty"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@google-cloud/vision": "^4.2.0",
|
||||
"@prisma/client": "^5.12.1",
|
||||
"axios": "^1.6.8",
|
||||
"dotenv": "^16.4.1",
|
||||
"lodash": "^4.17.21",
|
||||
"meilisearch": "^0.38.0",
|
||||
"mime-types": "^2.1.35",
|
||||
"minio": "^7.1.3",
|
||||
"pino": "^8.20.0",
|
||||
"pino-pretty": "^11.0.0",
|
||||
"sharp": "^0.33.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/lodash": "^4.17.0",
|
||||
"typescript": "^5.3.3",
|
||||
"esbuild": "^0.20.2",
|
||||
"esrun": "^3.2.26",
|
||||
"prisma": "^5.12.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "attachments" (
|
||||
"id" BIGINT NOT NULL,
|
||||
"msgId" BIGINT NOT NULL,
|
||||
"channelId" BIGINT NOT NULL,
|
||||
"memeId" UUID NOT NULL,
|
||||
"url" TEXT NOT NULL,
|
||||
"contentType" TEXT NOT NULL,
|
||||
"message" JSONB NOT NULL,
|
||||
"guild" JSONB NOT NULL,
|
||||
"channel" JSONB NOT NULL,
|
||||
"vision" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "attachments_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "memes" (
|
||||
"id" UUID NOT NULL DEFAULT gen_random_uuid(),
|
||||
"url" TEXT NOT NULL,
|
||||
"text" TEXT NOT NULL,
|
||||
"tags" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "memes_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "attachments_memeId_key" ON "attachments"("memeId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "attachments_channelId_idx" ON "attachments"("channelId");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "attachments" ADD CONSTRAINT "attachments_memeId_fkey" FOREIGN KEY ("memeId") REFERENCES "memes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,11 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- A unique constraint covering the columns `[attachmentId]` on the table `memes` will be added. If there are existing duplicate values, this will fail.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "memes" ADD COLUMN "attachmentId" BIGINT;
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "memes_attachmentId_key" ON "memes"("attachmentId");
|
||||
@@ -0,0 +1,9 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "channelhistory" (
|
||||
"id" BIGINT NOT NULL,
|
||||
"earliestMessageId" BIGINT NOT NULL,
|
||||
"latestMessageId" BIGINT NOT NULL,
|
||||
"backfillComplete" BOOLEAN NOT NULL DEFAULT false,
|
||||
|
||||
CONSTRAINT "channelhistory_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `backfillComplete` on the `channelhistory` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "channelhistory" DROP COLUMN "backfillComplete";
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- Added the required column `postedAt` to the `attachments` table without a default value. This is not possible if the table is not empty.
|
||||
- Added the required column `timestamp` to the `memes` table without a default value. This is not possible if the table is not empty.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "attachments"
|
||||
ADD COLUMN "postedAt"
|
||||
TIMESTAMP(3);
|
||||
|
||||
UPDATE "attachments"
|
||||
set "postedAt" = TIMESTAMP 'epoch' + ((id >> 22) + 1420070400000) * INTERVAL '1 millisecond';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "memes"
|
||||
ADD COLUMN "postedAt"
|
||||
TIMESTAMP(3);
|
||||
|
||||
UPDATE "memes"
|
||||
set "postedAt" = TIMESTAMP 'epoch' + (("attachmentId" >> 22) + 1420070400000) * INTERVAL '1 millisecond';
|
||||
|
||||
ALTER TABLE "attachments" ALTER COLUMN "postedAt" SET NOT NULL;
|
||||
ALTER TABLE "memes" ALTER COLUMN "postedAt" SET NOT NULL;
|
||||
@@ -0,0 +1,3 @@
|
||||
# Please do not edit this file manually
|
||||
# It should be added in your version-control system (i.e. Git)
|
||||
provider = "postgresql"
|
||||
@@ -0,0 +1,66 @@
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
// include for node docker image
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x"]
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("POSTGRES_URL")
|
||||
}
|
||||
|
||||
model Attachment {
|
||||
id BigInt @id
|
||||
msgId BigInt
|
||||
channelId BigInt
|
||||
|
||||
memeId String @unique @db.Uuid
|
||||
meme Meme @relation(fields: [memeId], references: [id], onDelete: Restrict)
|
||||
|
||||
url String
|
||||
contentType String
|
||||
|
||||
message Json
|
||||
guild Json
|
||||
channel Json
|
||||
vision Json
|
||||
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @default(now())
|
||||
postedAt DateTime
|
||||
|
||||
@@index([channelId])
|
||||
@@map("attachments")
|
||||
}
|
||||
|
||||
model Meme {
|
||||
id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
|
||||
url String
|
||||
text String
|
||||
tags Json
|
||||
|
||||
// need an timestamp for giving to meili to sort by
|
||||
// i think a usize would actually work but can't
|
||||
// convince the meili-js lib to serialize it properly to json
|
||||
// this one actually has to be .valueOf()'d before we send it
|
||||
// cause js numbers are bigger than postgres int4
|
||||
postedAt DateTime
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @default(now())
|
||||
|
||||
// required to prevent the same attachment being uploaded multiple times
|
||||
// nullable because maybe will have memes that don't come from a discord attachment
|
||||
attachmentId BigInt? @unique
|
||||
attachment Attachment?
|
||||
|
||||
@@map("memes")
|
||||
}
|
||||
|
||||
model ChannelHistory {
|
||||
id BigInt @id
|
||||
earliestMessageId BigInt
|
||||
latestMessageId BigInt
|
||||
|
||||
// we do this because you have to always wrap table names in "" if they have capitals
|
||||
@@map("channelhistory")
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import sharp from 'sharp'
|
||||
import axios from 'axios'
|
||||
import mime from 'mime-types'
|
||||
|
||||
import _ from 'lodash';
|
||||
|
||||
import type { Logger } from 'pino'
|
||||
|
||||
import type Minio from 'minio'
|
||||
import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client'
|
||||
|
||||
import type { MemeService } from './memes'
|
||||
|
||||
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
||||
|
||||
export const IMAGE_TYPES = [
|
||||
'image/png',
|
||||
'image/jpeg',
|
||||
'image/webp',
|
||||
'image/gif',
|
||||
]
|
||||
|
||||
export const VIDEO_TYPES = [
|
||||
'video/mp4',
|
||||
'video/quicktime'
|
||||
]
|
||||
|
||||
export function isImage(a: Attachment) {
|
||||
return IMAGE_TYPES.includes(a.contentType)
|
||||
}
|
||||
|
||||
export function objectName(a: Attachment) {
|
||||
if (isImage(a))
|
||||
return `${a.memeId}.webp`
|
||||
return `${a.memeId}.${mime.extension(a.contentType)}`
|
||||
}
|
||||
|
||||
|
||||
export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) {
|
||||
async function ensureStorageBucketExists() {
|
||||
const exists = await minio.bucketExists(STORAGE_BUCKET)
|
||||
if (exists) return true
|
||||
await minio.makeBucket(STORAGE_BUCKET)
|
||||
}
|
||||
|
||||
async function storeAttachment(a: Attachment) {
|
||||
const download = await axios.get(a.url, { responseType: 'stream' })
|
||||
|
||||
const stream = () => {
|
||||
if (isImage(a)) {
|
||||
const converter = sharp().webp()
|
||||
download.data.pipe(converter)
|
||||
return converter
|
||||
}
|
||||
return download.data
|
||||
}
|
||||
|
||||
return minio.putObject(STORAGE_BUCKET,
|
||||
objectName(a),
|
||||
stream(),
|
||||
null,
|
||||
// hardcode this for now, we upload everything as webp
|
||||
{ 'Content-Type': 'image/webp' })
|
||||
}
|
||||
|
||||
async function deleteAttachment(a: Attachment) {
|
||||
return minio.removeObject(STORAGE_BUCKET, objectName(a))
|
||||
}
|
||||
|
||||
async function process(a: Attachment, history: Partial<ChannelHistory>) {
|
||||
try {
|
||||
const results = await prisma.$transaction(async prisma => {
|
||||
const meme = memes.fromAttachment(a)
|
||||
const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
|
||||
const dbA = await prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
|
||||
log.debug({ msg: 'pending addition to db', dbMeme })
|
||||
|
||||
const upload = await storeAttachment(a)
|
||||
log.debug({ msg: 'successfully uploaded', upload })
|
||||
|
||||
await memes.persist(dbMeme)
|
||||
|
||||
log.info(dbMeme, 'new meme')
|
||||
return { meme: dbMeme, upload, attachment: dbA }
|
||||
}, { timeout: 60000 })
|
||||
} catch (e) {
|
||||
|
||||
// this happens pretty regularly it seems
|
||||
// when ppl reply to an attachment it is included in their message
|
||||
if (e.meta?.modelName !== 'Meme' || e.meta?.target?.[0] !== 'attachmentId') {
|
||||
log.error(e, 'failed to process attachment')
|
||||
log.error(_.omit(a, ['vision']), 'failed to process attachment')
|
||||
}
|
||||
|
||||
// deletes the upload if we got that far
|
||||
// (probably not)
|
||||
await deleteAttachment(a)
|
||||
}
|
||||
|
||||
// we assume here that a failure in processing was caused by a duplicate
|
||||
// so we update pg to say we've processed it
|
||||
const newHistory = await prisma.channelHistory.update({ data: history, where: { id: history.id }})
|
||||
log.debug({ newHistory }, 'updated channel history')
|
||||
return true
|
||||
}
|
||||
|
||||
await ensureStorageBucketExists()
|
||||
|
||||
return {
|
||||
process,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export type AttachmentService = Awaited<ReturnType<typeof attachmentService>>
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { PrismaClient, Attachment, Meme } from '@prisma/client';
|
||||
import type { Meilisearch } from 'meilisearch';
|
||||
import type { Logger } from 'pino';
|
||||
|
||||
import _ from 'lodash'
|
||||
|
||||
import * as attachments from './attachments'
|
||||
|
||||
const STORAGE_BUCKET = process.env.STORAGE_BUCKET
|
||||
|
||||
function memeUrl(a: Attachment) {
|
||||
const STORAGE_URL = process.env.STORAGE_URL
|
||||
const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
|
||||
return u.href
|
||||
}
|
||||
|
||||
export default async function memeService(baseLog: Logger, prisma: PrismaClient, meili: Meilisearch) {
|
||||
const log = baseLog.child({ service: 'meme' })
|
||||
|
||||
function fromAttachment(a: Attachment) {
|
||||
const text = a.vision.fullTextAnnotation?.text || ''
|
||||
const tags = [
|
||||
a.guild.name,
|
||||
a.channel.name,
|
||||
a.message.author.username,
|
||||
]
|
||||
|
||||
const m: Meme = {
|
||||
id: a.memeId,
|
||||
url: memeUrl(a),
|
||||
text,
|
||||
tags: tags,
|
||||
attachmentId: a.id,
|
||||
postedAt: a.postedAt,
|
||||
createdAt: undefined,
|
||||
updatedAt: undefined
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
async function ensureMeiliIndex() {
|
||||
const task = await meili.createIndex('memes', { primaryKey: 'id' })
|
||||
const result = await meili.waitForTask(task.taskUid)
|
||||
log.debug(result);
|
||||
const index = await meili.index('memes')
|
||||
await index.updateSortableAttributes(['postedAt'])
|
||||
return index;
|
||||
}
|
||||
|
||||
// BLOWS IT UP
|
||||
async function resetMeili() {
|
||||
const delTask = await meili.deleteIndex('memes')
|
||||
const delResult = await meili.waitForTask(delTask.taskUid)
|
||||
const index = await ensureMeiliIndex()
|
||||
const memes = (await prisma.meme.findMany())
|
||||
.map(m => ({ ...m, postedAt: m.postedAt.valueOf() }));
|
||||
|
||||
for await (const batch of _.chunk(memes, 1000)) {
|
||||
const task = await index.updateDocuments(batch)
|
||||
const res = await meili.waitForTask(task.taskUid)
|
||||
log.info({ res }, 'added meme batch')
|
||||
}
|
||||
}
|
||||
|
||||
async function persist(m: Meme) {
|
||||
const index = meili.index('memes')
|
||||
const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
|
||||
return index.updateDocuments([doc])
|
||||
}
|
||||
|
||||
await ensureMeiliIndex();
|
||||
|
||||
// BAM
|
||||
// await resetMeili();
|
||||
|
||||
return {
|
||||
ensureMeiliIndex,
|
||||
resetMeili,
|
||||
fromAttachment,
|
||||
persist,
|
||||
}
|
||||
}
|
||||
|
||||
export type MemeService = Awaited<ReturnType<typeof memeService>>
|
||||
@@ -0,0 +1,213 @@
|
||||
import { setTimeout } from 'timers/promises'
|
||||
import axios from 'axios'
|
||||
|
||||
import _ from 'lodash';
|
||||
|
||||
import type { PrismaClient, Attachment } from '@prisma/client';
|
||||
import type { ImageAnnotatorClient } from '@google-cloud/vision';
|
||||
import type { Logger } from 'pino';
|
||||
import type Minio from 'minio'
|
||||
|
||||
import sharp from 'sharp';
|
||||
|
||||
import type { AttachmentService } from './attachments'
|
||||
import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
|
||||
|
||||
// TODO
|
||||
// also fetch embeds and their descriptions
|
||||
|
||||
function convertSnowflakeToDate(snowflake) {
|
||||
const DISCORD_EPOCH = 1420070400000
|
||||
const milliseconds = global.BigInt(snowflake) >> 22n
|
||||
return new Date(Number(milliseconds) + DISCORD_EPOCH)
|
||||
}
|
||||
|
||||
export default async function scraper(
|
||||
discordAuth,
|
||||
vision: ImageAnnotatorClient,
|
||||
minio: Minio.Client,
|
||||
prisma: PrismaClient,
|
||||
baseLog: Logger,
|
||||
attachments: AttachmentService
|
||||
) {
|
||||
const headers = {
|
||||
'Authorization': discordAuth,
|
||||
}
|
||||
|
||||
async function fetchChannelMessages(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages&limit=100`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
// returns msgs in desc id order
|
||||
async function fetchChannelMessagesAfter(channelId, lastMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?after=${lastMessageId}&limit=100`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function fetchChannelMessagesBefore(channelId, earliestMessageId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}/messages?before=${earliestMessageId}&limit=100`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function fetchGuildDetails(guildId) {
|
||||
const url = `https://discord.com/api/v9/guilds/${guildId}`
|
||||
const res = await axios.get(url, { headers })
|
||||
return _.omit(res.data, ['emojis', 'stickers', 'roles', 'features'])
|
||||
}
|
||||
|
||||
async function fetchChannel(channelId) {
|
||||
const url = `https://discord.com/api/v9/channels/${channelId}`
|
||||
const res = await axios.get(url, { headers })
|
||||
return res.data
|
||||
}
|
||||
|
||||
async function detectText(attachment: Attachment) {
|
||||
// bail out of anything not an image
|
||||
if (!IMAGE_TYPES.includes(attachment.contentType)) return {}
|
||||
|
||||
const annotateRequest = {
|
||||
image: {
|
||||
source: {
|
||||
imageUri: attachment.url
|
||||
}
|
||||
},
|
||||
features: [{ type: 'TEXT_DETECTION' }],
|
||||
}
|
||||
const res = await vision.annotateImage(annotateRequest)
|
||||
// comes back as an array
|
||||
return res[0] || {}
|
||||
}
|
||||
|
||||
async function fetchMissingVision() {
|
||||
const attachments = await prisma.attachment
|
||||
.findMany({ where: { vision: {}, contentType: { in: IMAGE_TYPES } }})
|
||||
baseLog.info({attachments: attachments.length }, 'missing vision attachments')
|
||||
}
|
||||
|
||||
async function processChannel(channelId) {
|
||||
const channel = await fetchChannel(channelId)
|
||||
const guild = await fetchGuildDetails(channel.guild_id)
|
||||
const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
|
||||
|
||||
async function filterAttachments(messages): Promise<Attachment[]> {
|
||||
const acceptedContentTypes = [
|
||||
...IMAGE_TYPES,
|
||||
// ...VIDEO_TYPES,
|
||||
]
|
||||
|
||||
const validAttachments = messages
|
||||
.filter(m => m.attachments)
|
||||
// bundle message and attachment together
|
||||
.flatMap(m => m.attachments.map(a => [a, m]))
|
||||
.filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
|
||||
|
||||
log.debug({ validAttachments: validAttachments.length }, 'attachments found in messages')
|
||||
|
||||
return Promise.all(validAttachments
|
||||
.map(async ([a, m]) => {
|
||||
const attachment: Attachment = {
|
||||
id: BigInt(a.id),
|
||||
msgId: BigInt(m.id),
|
||||
channelId: BigInt(m.channel_id),
|
||||
url: a.url,
|
||||
contentType: a.content_type,
|
||||
|
||||
// create this ahead of time to associate with our copy of the media
|
||||
memeId: crypto.randomUUID(),
|
||||
|
||||
vision: {}, // quick hack so that detectText can receive a full attachment
|
||||
guild,
|
||||
channel,
|
||||
message: m,
|
||||
postedAt: convertSnowflakeToDate(a.id),
|
||||
// ts asks for these to be included but they don't need to be set
|
||||
createdAt: undefined,
|
||||
updatedAt: undefined
|
||||
}
|
||||
|
||||
// actually assign vision here
|
||||
return { ...attachment, vision: await detectText(attachment) }
|
||||
}))
|
||||
}
|
||||
|
||||
async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
|
||||
// reverse the messages because we want to yield them in chronological order
|
||||
// the api call returns them most recent first
|
||||
const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
|
||||
if (messages.length === 0) {
|
||||
log.info('reached latest message')
|
||||
}
|
||||
|
||||
log.debug({ messages: messages.length, latestMessageId }, 'fetched new messages')
|
||||
yield* await filterAttachments(messages)
|
||||
}
|
||||
|
||||
async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
|
||||
const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
|
||||
if (messages.length === 0) {
|
||||
log.info('reached beginning of history')
|
||||
}
|
||||
|
||||
log.debug({ messages: messages.length, earliestMessageId }, 'fetched older messages')
|
||||
yield* await filterAttachments(messages)
|
||||
}
|
||||
|
||||
const ensureHistory = async () => {
|
||||
const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }})
|
||||
if (stored) return stored
|
||||
|
||||
const lastMessageId = channel.last_message_id
|
||||
return await prisma.channelHistory.create({ data:
|
||||
{ latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
|
||||
})
|
||||
}
|
||||
|
||||
const history = await ensureHistory()
|
||||
|
||||
async function processLoop() {
|
||||
const history = await ensureHistory()
|
||||
const newAttachments = fetchNew(channelId, log, history.latestMessageId)
|
||||
const oldAttachments = fetchOld(channelId, log, history.earliestMessageId)
|
||||
|
||||
let oldCount = 0;
|
||||
let newCount = 0;
|
||||
|
||||
for await (const attachment of newAttachments) {
|
||||
log.debug({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
|
||||
const newHistory = { id: history.id, latestMessageId: attachment.msgId }
|
||||
await attachments.process(attachment, newHistory)
|
||||
// pretty oldschool, the async iterable from the fetch fn does not have a count
|
||||
// so we just use a plain old count to see if we should wait
|
||||
oldCount++
|
||||
}
|
||||
|
||||
for await (const attachment of oldAttachments) {
|
||||
log.debug({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
|
||||
const newHistory = { id: history.id, earliestMessageId: attachment.msgId }
|
||||
await attachments.process(attachment, newHistory)
|
||||
newCount++
|
||||
}
|
||||
|
||||
log.info({ oldCount, newCount }, 'processed attachments')
|
||||
|
||||
const shouldWait = newCount === 0 && oldCount === 0
|
||||
const PROCESS_DELAY_MS = 1000 * 60 // 1 minute
|
||||
if (shouldWait) {
|
||||
log.info({ PROCESS_DELAY_MS }, 'waiting for new messages')
|
||||
await setTimeout(PROCESS_DELAY_MS)
|
||||
}
|
||||
|
||||
return processLoop()
|
||||
}
|
||||
|
||||
await processLoop();
|
||||
}
|
||||
|
||||
return {
|
||||
processChannel,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
export default {
|
||||
id: 1227338263132635217n,
|
||||
msgId: 1227338263010873456n,
|
||||
channelId: 367936426962190336n,
|
||||
url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
|
||||
contentType: "image/jpeg",
|
||||
message: {
|
||||
id: "1227338263010873456",
|
||||
type: 0,
|
||||
content: "",
|
||||
channel_id: "367936426962190336",
|
||||
author: {
|
||||
id: "505353587174408193",
|
||||
username: "tgcharlet",
|
||||
avatar: "e800ffab640eb8684020054301668f79",
|
||||
discriminator: "0",
|
||||
public_flags: 0,
|
||||
flags: 0,
|
||||
banner: null,
|
||||
accent_color: null,
|
||||
global_name: "Chehrlet",
|
||||
avatar_decoration_data: null,
|
||||
banner_color: null,
|
||||
clan: null
|
||||
},
|
||||
attachments: [
|
||||
{
|
||||
id: "1227338263132635217",
|
||||
filename: "WOW.jpg",
|
||||
size: 55508,
|
||||
url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
|
||||
proxy_url: "https://media.discordapp.net/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
|
||||
width: 1080,
|
||||
height: 1022,
|
||||
content_type: "image/jpeg",
|
||||
content_scan_version: 1,
|
||||
placeholder: "YPcFD4LU/YlNhpxWh4iLiJhn+iRpT8AH",
|
||||
placeholder_version: 1
|
||||
}
|
||||
]
|
||||
},
|
||||
memeId: "7456180d-fa94-4465-bcb6-ea80524c67e8",
|
||||
vision: {
|
||||
textAnnotations: [],
|
||||
localizedObjectAnnotations: [],
|
||||
safeSearchAnnotation: null,
|
||||
imagePropertiesAnnotation: null,
|
||||
error: null,
|
||||
cropHintsAnnotation: null,
|
||||
fullTextAnnotation: {
|
||||
text: "damn thats crazy"
|
||||
},
|
||||
webDetection: null,
|
||||
productSearchResults: null,
|
||||
context: null
|
||||
},
|
||||
guild: {
|
||||
id: "239638846688460802",
|
||||
name: "The Philosopher's Meme",
|
||||
icon: "bda0d70d3bb2ec4df2dee343029d990d",
|
||||
description: "An applied philosophical community of practice for creatives, intellectuals, researchers, and developers. https://tpm.ac",
|
||||
home_header: null,
|
||||
splash: "0d80f531b01ab5abc8d153386f2af3e6",
|
||||
discovery_splash: "b35170de857b5eb58b1065131afa0826",
|
||||
banner: null,
|
||||
owner_id: "103361901899038720",
|
||||
application_id: null,
|
||||
region: "us-south",
|
||||
afk_channel_id: null,
|
||||
afk_timeout: 300,
|
||||
system_channel_id: "934578834534182953",
|
||||
system_channel_flags: 0,
|
||||
widget_enabled: true,
|
||||
widget_channel_id: "301188489490202626",
|
||||
verification_level: 3,
|
||||
default_message_notifications: 1,
|
||||
mfa_level: 1,
|
||||
explicit_content_filter: 2,
|
||||
max_presences: null,
|
||||
max_members: 500000,
|
||||
max_stage_video_channel_users: 50,
|
||||
max_video_channel_users: 25,
|
||||
vanity_url_code: null,
|
||||
premium_tier: 1,
|
||||
premium_subscription_count: 3,
|
||||
preferred_locale: "en-US",
|
||||
rules_channel_id: "301188489490202626",
|
||||
safety_alerts_channel_id: "837815188970537031",
|
||||
public_updates_channel_id: "837815188970537031",
|
||||
hub_type: null,
|
||||
premium_progress_bar_enabled: true,
|
||||
latest_onboarding_question_id: "1136794072313507860",
|
||||
nsfw: false,
|
||||
nsfw_level: 0,
|
||||
incidents_data: null,
|
||||
inventory_settings: null,
|
||||
embed_enabled: true,
|
||||
embed_channel_id: "301188489490202626"
|
||||
},
|
||||
channel: {
|
||||
id: "367936426962190336",
|
||||
type: 0,
|
||||
last_message_id: "1227404638450483340",
|
||||
flags: 0,
|
||||
last_pin_timestamp: "2022-07-12T12:05:47+00:00",
|
||||
guild_id: "239638846688460802",
|
||||
name: "shitposts_memes",
|
||||
parent_id: "1023698856191471667",
|
||||
rate_limit_per_user: 0,
|
||||
topic: "memes, shitposts; memeing, shitposting",
|
||||
position: 38,
|
||||
nsfw: false,
|
||||
icon_emoji: {
|
||||
id: null,
|
||||
name: "🤪"
|
||||
},
|
||||
theme_color: null
|
||||
},
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node"
|
||||
},
|
||||
}
|
||||
Reference in new issue
Block a user