From 66228047f4543f9862f96e8ccee04f2cdf1a0fd1 Mon Sep 17 00:00:00 2001 From: Nathan Rashleigh Date: Wed, 10 Apr 2024 12:33:47 +1000 Subject: [PATCH] sublime fkt but everything working --- index.ts | 43 +++++-- memestream.sublime-project | 17 --- package-lock.json | 33 +++++ package.json | 4 +- .../20240410010107_init/migration.sql | 38 ++++++ .../migration.sql | 11 ++ prisma/migrations/migration_lock.toml | 3 + prisma/schema.prisma | 7 +- src/attachments.ts | 71 ++++++++++ src/memes.ts | 31 +++++ src/scraper.ts | 107 +++------------- src/test.attachment.ts | 121 ++++++++++++++++++ 12 files changed, 363 insertions(+), 123 deletions(-) delete mode 100644 memestream.sublime-project create mode 100644 prisma/migrations/20240410010107_init/migration.sql create mode 100644 prisma/migrations/20240410015301_memes_attachment_id/migration.sql create mode 100644 prisma/migrations/migration_lock.toml create mode 100644 src/attachments.ts create mode 100644 src/memes.ts create mode 100644 src/test.attachment.ts diff --git a/index.ts b/index.ts index 49a474b..b1530ce 100644 --- a/index.ts +++ b/index.ts @@ -1,40 +1,57 @@ import "dotenv/config" +import fs from 'fs/promises' + import * as Minio from 'minio' import Vision from '@google-cloud/vision' import { PrismaClient } from '@prisma/client' +import { MeiliSearch } from 'meilisearch' import discordScraper from './src/scraper' +import testAttachment from "./src/test.attachment" +import attachmentService from "./src/attachments" + +// @ts-ignore - ts don't know wtf it talking about +BigInt.prototype.toJSON = function bigIntStringify() { + return this.toString(); +}; const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default const STORAGE_BUCKET = process.env.STORAGE_BUCKET +const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) +const prisma = new PrismaClient() const minio = new Minio.Client({ endPoint: process.env.STORAGE_URL, accessKey: process.env.STORAGE_ACCESS_KEY, secretKey: process.env.STORAGE_SECRET_KEY, }) -const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS }) -const prisma = new PrismaClient() +const client = new MeiliSearch({ + host: process.env.MEILI_URL, + apiKey: process.env.MEILI_MASTER_KEY, +}) -async function storageBucketExists() { +const attachments = attachmentService(minio, prisma) +const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio) + +async function ensureStorageBucketExists() { const exists = await minio.bucketExists(STORAGE_BUCKET) if (exists) return true await minio.makeBucket(STORAGE_BUCKET) } -console.log(await storageBucketExists()) +async function processChannels(channelIds) { + await ensureStorageBucketExists() + const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c))) + const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a))) + console.log(results); + return results +} -// BIGINTS -// must be strings const CHANNEL_IDS = [ - '367936426962190336', // TPM shitposts - '1095060717117374687', // aella memes + 367936426962190336n, // TPM shitposts + 1095060717117374687n, // aella memes ] -const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio) - -const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c))) -console.log(msgs) -const downloads = await Promise.all(msgs.flat().map(scraper.downloadAttachment)) +await processChannels(CHANNEL_IDS) diff --git a/memestream.sublime-project b/memestream.sublime-project deleted file mode 100644 index 4cc4415..0000000 --- a/memestream.sublime-project +++ /dev/null @@ -1,17 +0,0 @@ -{ - "folders": - [ - { - "path": ".", - }, - { - "path": "/home/ntr/memestream" - }, - ], - "build_systems": [ - { - "name": "Memestream - npm start", - "cmd": ["npm", "start"], - }, - ], -} diff --git a/package-lock.json b/package-lock.json index 9475e40..20f6d68 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@prisma/client": "^5.12.1", "axios": "^1.6.8", "dotenv": "^16.4.1", + "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", "sharp": "^0.33.3", @@ -1328,6 +1329,14 @@ "node": ">= 0.8" } }, + "node_modules/cross-fetch": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -2043,6 +2052,14 @@ "node": ">=10" } }, + "node_modules/meilisearch": { + "version": "0.38.0", + "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz", + "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==", + "dependencies": { + "cross-fetch": "^3.1.6" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -3392,6 +3409,14 @@ "delayed-stream": "~1.0.0" } }, + "cross-fetch": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "requires": { + "node-fetch": "^2.6.12" + } + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -3892,6 +3917,14 @@ "yallist": "^4.0.0" } }, + "meilisearch": { + "version": "0.38.0", + "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz", + "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==", + "requires": { + "cross-fetch": "^3.1.6" + } + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", diff --git a/package.json b/package.json index 2e38fd8..3f4755f 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "index.ts", "type": "module", "scripts": { - "start": "esrun --watch index.ts", + "start": "esrun index.ts", + "dev": "esrun --watch index.ts", "migrate:make": "prisma migrate dev --name", "test": "echo \"Error: no test specified\" && exit 1" }, @@ -16,6 +17,7 @@ "@prisma/client": "^5.12.1", "axios": "^1.6.8", "dotenv": "^16.4.1", + "meilisearch": "^0.38.0", "mime-types": "^2.1.35", "minio": "^7.1.3", "sharp": "^0.33.3", diff --git a/prisma/migrations/20240410010107_init/migration.sql b/prisma/migrations/20240410010107_init/migration.sql new file mode 100644 index 0000000..a7e3e05 --- /dev/null +++ b/prisma/migrations/20240410010107_init/migration.sql @@ -0,0 +1,38 @@ +-- CreateTable +CREATE TABLE "attachments" ( + "id" BIGINT NOT NULL, + "msgId" BIGINT NOT NULL, + "channelId" BIGINT NOT NULL, + "memeId" UUID NOT NULL, + "url" TEXT NOT NULL, + "contentType" TEXT NOT NULL, + "message" JSONB NOT NULL, + "guild" JSONB NOT NULL, + "channel" JSONB NOT NULL, + "vision" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "attachments_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "memes" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "url" TEXT NOT NULL, + "text" TEXT NOT NULL, + "tags" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "memes_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "attachments_memeId_key" ON "attachments"("memeId"); + +-- CreateIndex +CREATE INDEX "attachments_channelId_idx" ON "attachments"("channelId"); + +-- AddForeignKey +ALTER TABLE "attachments" ADD CONSTRAINT "attachments_memeId_fkey" FOREIGN KEY ("memeId") REFERENCES "memes"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/20240410015301_memes_attachment_id/migration.sql b/prisma/migrations/20240410015301_memes_attachment_id/migration.sql new file mode 100644 index 0000000..ff55371 --- /dev/null +++ b/prisma/migrations/20240410015301_memes_attachment_id/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - A unique constraint covering the columns `[attachmentId]` on the table `memes` will be added. If there are existing duplicate values, this will fail. + +*/ +-- AlterTable +ALTER TABLE "memes" ADD COLUMN "attachmentId" BIGINT; + +-- CreateIndex +CREATE UNIQUE INDEX "memes_attachmentId_key" ON "memes"("attachmentId"); diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..fbffa92 --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a127572..03e70bd 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -26,7 +26,7 @@ model Attachment { vision Json createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + updatedAt DateTime @default(now()) @@index([channelId]) @@map("attachments") @@ -39,8 +39,11 @@ model Meme { tags Json createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + updatedAt DateTime @default(now()) + // required to prevent the same attachment being uploaded multiple times + // nullable because maybe will have memes that don't come from a discord attachment + attachmentId BigInt? @unique attachment Attachment? @@map("memes") diff --git a/src/attachments.ts b/src/attachments.ts new file mode 100644 index 0000000..dd55744 --- /dev/null +++ b/src/attachments.ts @@ -0,0 +1,71 @@ +import sharp from 'sharp'; +import axios from 'axios' +import mime from 'mime-types' + +import type Minio from 'minio' +import type { PrismaClient, Attachment } from '@prisma/client'; + +import * as memes from './memes' + +const STORAGE_BUCKET = process.env.STORAGE_BUCKET + +export const IMAGE_TYPES = [ + 'image/png', + 'image/jpeg', + 'image/webp', + 'image/gif', +] + +export const VIDEO_TYPES = [ + 'video/mp4', + 'video/quicktime' +] + +export function isImage(a: Attachment) { + return IMAGE_TYPES.includes(a.contentType) +} + +export function objectName(a: Attachment) { + if (isImage(a)) + return `${a.memeId}.webp` + return `${a.memeId}.${mime.extension(a.contentType)}` +} + +export default function attachmentService(minio: Minio.Client, prisma: PrismaClient) { + async function storeAttachment(a: Attachment) { + const download = await axios.get(a.url, { responseType: 'stream' }) + + const stream = () => { + if (isImage(a)) { + const converter = sharp().webp() + download.data.pipe(converter) + return converter + } + return download.data + } + + return minio.putObject(STORAGE_BUCKET, objectName(a), stream()) + } + + async function deleteAttachment(a: Attachment) { + return minio.removeObject(STORAGE_BUCKET, objectName(a)) + } + + async function process(a: Attachment) { + const meme = memes.fromAttachment(a) + const upload = await storeAttachment(a) + return prisma.$transaction(async prisma => { + const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } }) + const dbA = prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } }) + console.log('pending addition to db', dbMeme) + const upload = await storeAttachment(a) + console.log('successfully uploaded', upload) + return { meme: dbMeme, upload, attachment: dbA } + }) + } + + return { + process, + } +} + diff --git a/src/memes.ts b/src/memes.ts new file mode 100644 index 0000000..b5ff226 --- /dev/null +++ b/src/memes.ts @@ -0,0 +1,31 @@ +import type { PrismaClient, Attachment, Meme } from '@prisma/client'; + +import * as attachments from './attachments' + +function memeUrl(a: Attachment) { + const STORAGE_URL = process.env.STORAGE_URL + const path = attachments.objectName(a) + const u = new URL(path, `https://${STORAGE_URL}`) + return u.href +} + +export function fromAttachment(a: Attachment) { + const text = a.vision.fullTextAnnotation?.text || '' + const tags = [ + a.guild.name, + a.channel.name, + a.message.author.username, + ] + + const m: Meme = { + id: a.memeId, + url: memeUrl(a), + text, + tags: tags, + attachmentId: a.id, + createdAt: undefined, + updatedAt: undefined + } + + return m +} diff --git a/src/scraper.ts b/src/scraper.ts index 185f1bb..9e45ab8 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -1,7 +1,4 @@ -import path from 'path' - import axios from 'axios' -import mime from 'mime-types' import type { PrismaClient, Attachment } from '@prisma/client'; import type { ImageAnnotatorClient } from '@google-cloud/vision'; @@ -9,21 +6,14 @@ import type Minio from 'minio' import sharp from 'sharp'; -const IMAGE_TYPES = [ - 'image/png', - 'image/jpeg', - 'image/webp', - 'image/gif', -] +import * as attachments from './attachments' -const VIDEO_TYPES = [ - 'video/mp4', - 'video/quicktime' -] - -const STORAGE_BUCKET = process.env.STORAGE_BUCKET - -export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client) { +export default function scraper( +discordAuth, +vision: ImageAnnotatorClient, +minio: Minio.Client, +prisma: PrismaClient, +) { const headers = { 'Authorization': discordAuth, } @@ -50,7 +40,7 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio async function detectText(attachment: Attachment) { // bail out of anything not an image - if (!IMAGE_TYPES.includes(attachment.contentType)) return {} + if (!attachments.IMAGE_TYPES.includes(attachment.contentType)) return {} const annotateRequest = { image: { @@ -65,41 +55,24 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio return res[0] || {} } - async function downloadAttachment(a: Attachment) { - const download = await axios.get(a.url, { responseType: 'stream' }) - const isImage = IMAGE_TYPES.includes(a.contentType) - - const objectName = () => { - if (isImage) - return `${a.memeId}.webp` - - return `${a.memeId}.${mime.extension(a.contentType)}` - } - - const stream = async () => { - if (isImage) { - const converter = sharp().webp() - download.data.pipe(converter) - return converter - } - return download.data - } - - return minio.putObject(STORAGE_BUCKET, objectName(), await stream()) - } - function filterAttachments(messages, guild, channel) { const acceptedContentTypes = [ - ...IMAGE_TYPES, - ...VIDEO_TYPES, + ...attachments.IMAGE_TYPES, + ...attachments.VIDEO_TYPES, ] - return Promise.all(messages + const validAttachments = messages .filter(m => m.attachments) // bundle message and attachment together .map(m => m.attachments.map(a => [a, m])) .flat() .filter(([a, m]) => acceptedContentTypes.includes(a.content_type)) + + const attachmentIds = validAttachments.map(([a, m]) => a.id) + + const alreadyProcessedIds = prisma.attachment.find + + return Promise.all(validAttachments .map(async ([a, m]) => { const attachment: Attachment = { id: BigInt(a.id), @@ -145,51 +118,5 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio return { getChannelAttachments, - downloadAttachment, } } - - - // async def download_attachment(attachment: Attachment): - // async with http_session.get(attachment.url) as response: - // if response.status == 200: - // (filename, ext) = attachment.file_details() - // object_name = attachment.object_name() - - // print(f"uploading object_name={object_name} filename={filename} ext={ext}") - // res_bytes = await response.read() - - // # make a webp conversion for obvious image types - // if attachment.convert_to_webp(): - // # have to read the whole img before converting to webp - // image = Image.open(BytesIO(res_bytes)) - // webp_output = BytesIO() - // image.save(webp_output, format='WebP') - // webp_output.seek(0) - // return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output) - - // return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes) - - // def create_attachments(message, channel, guild): - // urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']] - - // def generate_attachment(url: Url, attachmentId: str): - // if not url.path: - // return None - - // (filename, ext) = os.path.splitext(os.path.basename(url.path)) - // extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4'] - // if ext.lower()[1:] not in extensions: - // return None - - // return Attachment( - // id=uuid4(), - // attachmentId=int(attachmentId), - // msgId=int(message['id']), - // message=message, - // guild=guild, - // url=url.url, - // channel=channel - // ) - - // return [generate_attachment(*url) for url in urls] \ No newline at end of file diff --git a/src/test.attachment.ts b/src/test.attachment.ts new file mode 100644 index 0000000..84e9499 --- /dev/null +++ b/src/test.attachment.ts @@ -0,0 +1,121 @@ +export default { + id: 1227338263132635217n, + msgId: 1227338263010873456n, + channelId: 367936426962190336n, + url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&", + contentType: "image/jpeg", + message: { + id: "1227338263010873456", + type: 0, + content: "", + channel_id: "367936426962190336", + author: { + id: "505353587174408193", + username: "tgcharlet", + avatar: "e800ffab640eb8684020054301668f79", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "Chehrlet", + avatar_decoration_data: null, + banner_color: null, + clan: null + }, + attachments: [ + { + id: "1227338263132635217", + filename: "WOW.jpg", + size: 55508, + url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&", + proxy_url: "https://media.discordapp.net/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&", + width: 1080, + height: 1022, + content_type: "image/jpeg", + content_scan_version: 1, + placeholder: "YPcFD4LU/YlNhpxWh4iLiJhn+iRpT8AH", + placeholder_version: 1 + } + ] + }, + memeId: "7456180d-fa94-4465-bcb6-ea80524c67e8", + vision: { + textAnnotations: [], + localizedObjectAnnotations: [], + safeSearchAnnotation: null, + imagePropertiesAnnotation: null, + error: null, + cropHintsAnnotation: null, + fullTextAnnotation: { + text: "damn thats crazy" + }, + webDetection: null, + productSearchResults: null, + context: null + }, + guild: { + id: "239638846688460802", + name: "The Philosopher's Meme", + icon: "bda0d70d3bb2ec4df2dee343029d990d", + description: "An applied philosophical community of practice for creatives, intellectuals, researchers, and developers. https://tpm.ac", + home_header: null, + splash: "0d80f531b01ab5abc8d153386f2af3e6", + discovery_splash: "b35170de857b5eb58b1065131afa0826", + banner: null, + owner_id: "103361901899038720", + application_id: null, + region: "us-south", + afk_channel_id: null, + afk_timeout: 300, + system_channel_id: "934578834534182953", + system_channel_flags: 0, + widget_enabled: true, + widget_channel_id: "301188489490202626", + verification_level: 3, + default_message_notifications: 1, + mfa_level: 1, + explicit_content_filter: 2, + max_presences: null, + max_members: 500000, + max_stage_video_channel_users: 50, + max_video_channel_users: 25, + vanity_url_code: null, + premium_tier: 1, + premium_subscription_count: 3, + preferred_locale: "en-US", + rules_channel_id: "301188489490202626", + safety_alerts_channel_id: "837815188970537031", + public_updates_channel_id: "837815188970537031", + hub_type: null, + premium_progress_bar_enabled: true, + latest_onboarding_question_id: "1136794072313507860", + nsfw: false, + nsfw_level: 0, + incidents_data: null, + inventory_settings: null, + embed_enabled: true, + embed_channel_id: "301188489490202626" + }, + channel: { + id: "367936426962190336", + type: 0, + last_message_id: "1227404638450483340", + flags: 0, + last_pin_timestamp: "2022-07-12T12:05:47+00:00", + guild_id: "239638846688460802", + name: "shitposts_memes", + parent_id: "1023698856191471667", + rate_limit_per_user: 0, + topic: "memes, shitposts; memeing, shitposting", + position: 38, + nsfw: false, + icon_emoji: { + id: null, + name: "🤪" + }, + theme_color: null + }, + createdAt: new Date(), + updatedAt: new Date(), +}