sublime fkt but everything working

index.ts

@@ -1,40 +1,57 @@
 import "dotenv/config"
 
+import fs from 'fs/promises'
+
 import * as Minio from 'minio'
 import Vision from '@google-cloud/vision'
 import { PrismaClient } from '@prisma/client'
+import { MeiliSearch } from 'meilisearch'
 
 import discordScraper from './src/scraper'
+import testAttachment from "./src/test.attachment"
+import attachmentService from "./src/attachments"
+
+// @ts-ignore - ts don't know wtf it talking about
+BigInt.prototype.toJSON = function bigIntStringify() {
+    return this.toString();
+};
 
 const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { assert: { type: "json" } })).default
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
 
+const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
+const prisma = new PrismaClient()
 const minio = new Minio.Client({
     endPoint: process.env.STORAGE_URL,
     accessKey: process.env.STORAGE_ACCESS_KEY,
     secretKey: process.env.STORAGE_SECRET_KEY,
 })
-const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
+const client = new MeiliSearch({
-const prisma = new PrismaClient()
+  host: process.env.MEILI_URL,
+  apiKey: process.env.MEILI_MASTER_KEY,
+})
 
-async function storageBucketExists() {
+const attachments = attachmentService(minio, prisma)
+const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
+
+async function ensureStorageBucketExists() {
     const exists = await minio.bucketExists(STORAGE_BUCKET)
     if (exists) return true
     await minio.makeBucket(STORAGE_BUCKET)
 }
 
-console.log(await storageBucketExists())
+async function processChannels(channelIds) {
+    await ensureStorageBucketExists()
+    const msgs = await Promise.all(channelIds.map(c => scraper.getChannelAttachments(c)))
+    const results = await Promise.allSettled(msgs.flat().map(a => attachments.process(a)))
+    console.log(results);
+    return results
+}
 
-// BIGINTS
-// must be strings
 const CHANNEL_IDS = [
-    '367936426962190336',  // TPM shitposts
+    367936426962190336n,  // TPM shitposts
-    '1095060717117374687', // aella memes
+    1095060717117374687n, // aella memes
 ]
 
-const scraper = discordScraper(process.env.DISCORD_AUTH, vision, minio)
+await processChannels(CHANNEL_IDS)
-
-const msgs = await Promise.all(CHANNEL_IDS.map(c => scraper.getChannelAttachments(c)))
-console.log(msgs)
-const downloads = await Promise.all(msgs.flat().map(scraper.downloadAttachment))
 

memestream.sublime-project

@@ -1,17 +0,0 @@
-{
-	"folders":
-	[
-        {
-            "path": ".",
-        },
-        {
-			"path": "/home/ntr/memestream"
-		},
-    ],
-    "build_systems": [
-        {
-            "name": "Memestream - npm start",
-            "cmd": ["npm", "start"],
-        },
-    ],
-}

package-lock.json

@@ -13,6 +13,7 @@
         "@prisma/client": "^5.12.1",
         "axios": "^1.6.8",
         "dotenv": "^16.4.1",
+        "meilisearch": "^0.38.0",
         "mime-types": "^2.1.35",
         "minio": "^7.1.3",
         "sharp": "^0.33.3",
@@ -1328,6 +1329,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/cross-fetch": {
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
+      "dependencies": {
+        "node-fetch": "^2.6.12"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -2043,6 +2052,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/meilisearch": {
+      "version": "0.38.0",
+      "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz",
+      "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==",
+      "dependencies": {
+        "cross-fetch": "^3.1.6"
+      }
+    },
     "node_modules/mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -3392,6 +3409,14 @@
         "delayed-stream": "~1.0.0"
       }
     },
+    "cross-fetch": {
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
+      "requires": {
+        "node-fetch": "^2.6.12"
+      }
+    },
     "debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -3892,6 +3917,14 @@
         "yallist": "^4.0.0"
       }
     },
+    "meilisearch": {
+      "version": "0.38.0",
+      "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.38.0.tgz",
+      "integrity": "sha512-bHaq8nYxSKw9/Qslq1Zes5g9tHgFkxy/I9o8942wv2PqlNOT0CzptIkh/x98N52GikoSZOXSQkgt6oMjtf5uZw==",
+      "requires": {
+        "cross-fetch": "^3.1.6"
+      }
+    },
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",

package.json

@@ -5,7 +5,8 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "start": "esrun --watch index.ts",
+    "start": "esrun index.ts",
+    "dev": "esrun --watch index.ts",
     "migrate:make": "prisma migrate dev --name",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -16,6 +17,7 @@
     "@prisma/client": "^5.12.1",
     "axios": "^1.6.8",
     "dotenv": "^16.4.1",
+    "meilisearch": "^0.38.0",
     "mime-types": "^2.1.35",
     "minio": "^7.1.3",
     "sharp": "^0.33.3",

prisma/migrations/20240410010107_init/migration.sql

@@ -0,0 +1,38 @@
+-- CreateTable
+CREATE TABLE "attachments" (
+    "id" BIGINT NOT NULL,
+    "msgId" BIGINT NOT NULL,
+    "channelId" BIGINT NOT NULL,
+    "memeId" UUID NOT NULL,
+    "url" TEXT NOT NULL,
+    "contentType" TEXT NOT NULL,
+    "message" JSONB NOT NULL,
+    "guild" JSONB NOT NULL,
+    "channel" JSONB NOT NULL,
+    "vision" JSONB NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "attachments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "memes" (
+    "id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "url" TEXT NOT NULL,
+    "text" TEXT NOT NULL,
+    "tags" JSONB NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "memes_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "attachments_memeId_key" ON "attachments"("memeId");
+
+-- CreateIndex
+CREATE INDEX "attachments_channelId_idx" ON "attachments"("channelId");
+
+-- AddForeignKey
+ALTER TABLE "attachments" ADD CONSTRAINT "attachments_memeId_fkey" FOREIGN KEY ("memeId") REFERENCES "memes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20240410015301_memes_attachment_id/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[attachmentId]` on the table `memes` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- AlterTable
+ALTER TABLE "memes" ADD COLUMN     "attachmentId" BIGINT;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "memes_attachmentId_key" ON "memes"("attachmentId");

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

prisma/schema.prisma

@@ -26,7 +26,7 @@ model Attachment {
     vision    Json
 
     createdAt  DateTime @default(now())
-    updatedAt  DateTime @updatedAt
+    updatedAt  DateTime @default(now())
 
     @@index([channelId])
     @@map("attachments")
@@ -39,8 +39,11 @@ model Meme {
     tags       Json
 
     createdAt  DateTime @default(now())
-    updatedAt  DateTime @updatedAt
+    updatedAt  DateTime @default(now())
 
+    // required to prevent the same attachment being uploaded multiple times
+    // nullable because maybe will have memes that don't come from a discord attachment
+    attachmentId BigInt? @unique
     attachment Attachment?
 
     @@map("memes")

src/attachments.ts

@@ -0,0 +1,71 @@
+import sharp from 'sharp';
+import axios from 'axios'
+import mime from 'mime-types'
+
+import type Minio from 'minio'
+import type { PrismaClient, Attachment } from '@prisma/client';
+
+import * as memes from './memes'
+
+const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+
+export const IMAGE_TYPES = [
+    'image/png',
+    'image/jpeg',
+    'image/webp',
+    'image/gif',
+]
+
+export const VIDEO_TYPES = [
+    'video/mp4',
+    'video/quicktime'
+]
+
+export function isImage(a: Attachment) {
+    return IMAGE_TYPES.includes(a.contentType)
+}
+
+export function objectName(a: Attachment) {
+    if (isImage(a))
+        return `${a.memeId}.webp`
+    return `${a.memeId}.${mime.extension(a.contentType)}`
+}
+
+export default function attachmentService(minio: Minio.Client, prisma: PrismaClient) {
+    async function storeAttachment(a: Attachment) {
+        const download = await axios.get(a.url, { responseType: 'stream' })
+
+        const stream = () => {
+            if (isImage(a)) {
+                const converter = sharp().webp()
+                download.data.pipe(converter)
+                return converter
+            }
+            return download.data
+        }
+
+        return minio.putObject(STORAGE_BUCKET, objectName(a), stream())
+    }
+
+    async function deleteAttachment(a: Attachment) {
+        return minio.removeObject(STORAGE_BUCKET, objectName(a))
+    }
+
+    async function process(a: Attachment) {
+        const meme = memes.fromAttachment(a)
+        const upload = await storeAttachment(a)
+        return prisma.$transaction(async prisma => {
+            const dbMeme = await prisma.meme.upsert({ create: meme, update: meme, where: { id: meme.id } })
+            const dbA = prisma.attachment.upsert({ create: a, update: a, where: { id: a.id } })
+            console.log('pending addition to db', dbMeme)
+            const upload = await storeAttachment(a)
+            console.log('successfully uploaded', upload)
+            return { meme: dbMeme, upload, attachment: dbA }
+        })
+    }
+
+    return {
+        process,
+    }
+}
+

src/memes.ts

@@ -0,0 +1,31 @@
+import type { PrismaClient, Attachment, Meme } from '@prisma/client';
+
+import * as attachments from './attachments'
+
+function memeUrl(a: Attachment) {
+    const STORAGE_URL = process.env.STORAGE_URL
+    const path = attachments.objectName(a)
+    const u = new URL(path, `https://${STORAGE_URL}`)
+    return u.href
+}
+
+export function fromAttachment(a: Attachment) {
+    const text = a.vision.fullTextAnnotation?.text || ''
+    const tags = [
+        a.guild.name,
+        a.channel.name,
+        a.message.author.username,
+    ]
+
+    const m: Meme = {
+        id: a.memeId,
+        url: memeUrl(a),
+        text,
+        tags: tags,
+        attachmentId: a.id,
+        createdAt: undefined,
+        updatedAt: undefined
+    }
+
+    return m
+}

src/scraper.ts

@@ -1,7 +1,4 @@
-import path from 'path'
-
 import axios from 'axios'
-import mime from 'mime-types'
 
 import type { PrismaClient, Attachment } from '@prisma/client';
 import type { ImageAnnotatorClient } from '@google-cloud/vision';
@@ -9,21 +6,14 @@ import type Minio from 'minio'
 
 import sharp from 'sharp';
 
-const IMAGE_TYPES = [
+import * as attachments from './attachments'
-    'image/png',
-    'image/jpeg',
-    'image/webp',
-    'image/gif',
-]
 
-const VIDEO_TYPES = [
+export default function scraper(
-    'video/mp4',
+discordAuth,
-    'video/quicktime'
+vision: ImageAnnotatorClient,
-]
+minio: Minio.Client,
-
+prisma: PrismaClient,
-const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+) {
-
-export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio: Minio.Client) {
     const headers = {
         'Authorization': discordAuth,
     }
@@ -50,7 +40,7 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
 
     async function detectText(attachment: Attachment) {
         // bail out of anything not an image
-        if (!IMAGE_TYPES.includes(attachment.contentType)) return {}
+        if (!attachments.IMAGE_TYPES.includes(attachment.contentType)) return {}
 
         const annotateRequest = {
             image: {
@@ -65,41 +55,24 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
         return res[0] || {}
     }
 
-    async function downloadAttachment(a: Attachment) {
-        const download = await axios.get(a.url, { responseType: 'stream' })
-        const isImage = IMAGE_TYPES.includes(a.contentType)
-
-        const objectName = () => {
-            if (isImage)
-                return `${a.memeId}.webp`
-
-            return `${a.memeId}.${mime.extension(a.contentType)}`
-        }
-
-        const stream = async () => {
-            if (isImage) {
-                const converter = sharp().webp()
-                download.data.pipe(converter)
-                return converter
-            }
-            return download.data
-        }
-
-        return minio.putObject(STORAGE_BUCKET, objectName(), await stream())
-    }
-
     function filterAttachments(messages, guild, channel) {
         const acceptedContentTypes = [
-            ...IMAGE_TYPES,
+            ...attachments.IMAGE_TYPES,
-            ...VIDEO_TYPES,
+            ...attachments.VIDEO_TYPES,
         ]
 
-        return Promise.all(messages
+        const validAttachments = messages
             .filter(m => m.attachments)
             // bundle message and attachment together
             .map(m => m.attachments.map(a => [a, m]))
             .flat()
             .filter(([a, m]) => acceptedContentTypes.includes(a.content_type))
+
+        const attachmentIds = validAttachments.map(([a, m]) => a.id)
+
+        const alreadyProcessedIds = prisma.attachment.find
+
+        return Promise.all(validAttachments
             .map(async ([a, m]) => {
                 const attachment: Attachment = {
                     id: BigInt(a.id),
@@ -145,51 +118,5 @@ export default function scraper(discordAuth, vision: ImageAnnotatorClient, minio
 
     return {
         getChannelAttachments,
-        downloadAttachment,
     }
 }
-
-
-    // async def download_attachment(attachment: Attachment):
-    //     async with http_session.get(attachment.url) as response:
-    //         if response.status == 200:
-    //             (filename, ext) = attachment.file_details()
-    //             object_name = attachment.object_name()
-
-    //             print(f"uploading object_name={object_name} filename={filename} ext={ext}")
-    //             res_bytes = await response.read()
-
-    //             # make a webp conversion for obvious image types
-    //             if attachment.convert_to_webp():
-    //                 # have to read the whole img before converting to webp
-    //                 image = Image.open(BytesIO(res_bytes))
-    //                 webp_output = BytesIO()
-    //                 image.save(webp_output, format='WebP')
-    //                 webp_output.seek(0)
-    //                 return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=webp_output)
-
-    //             return await storage.put_object(Bucket=STORAGE_BUCKET, Key=object_name, Body=res_bytes)
-
-    // def create_attachments(message, channel, guild):
-    //     urls = [(parse_url(attachment['url']), attachment['id']) for attachment in message['attachments']]
-
-    //     def generate_attachment(url: Url, attachmentId: str):
-    //         if not url.path:
-    //             return None
-
-    //         (filename, ext) = os.path.splitext(os.path.basename(url.path))
-    //         extensions = ['jpg', 'png', 'jpeg', 'webp', 'mov', 'gif', 'mp4']
-    //         if ext.lower()[1:] not in extensions:
-    //             return None
-
-    //         return Attachment(
-    //             id=uuid4(),
-    //             attachmentId=int(attachmentId),
-    //             msgId=int(message['id']),
-    //             message=message,
-    //             guild=guild,
-    //             url=url.url,
-    //             channel=channel
-    //         )
-
-    //     return [generate_attachment(*url) for url in urls]

src/test.attachment.ts

@@ -0,0 +1,121 @@
+export default {
+  id: 1227338263132635217n,
+  msgId: 1227338263010873456n,
+  channelId: 367936426962190336n,
+  url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+  contentType: "image/jpeg",
+  message: {
+    id: "1227338263010873456",
+    type: 0,
+    content: "",
+    channel_id: "367936426962190336",
+    author: {
+      id: "505353587174408193",
+      username: "tgcharlet",
+      avatar: "e800ffab640eb8684020054301668f79",
+      discriminator: "0",
+      public_flags: 0,
+      flags: 0,
+      banner: null,
+      accent_color: null,
+      global_name: "Chehrlet",
+      avatar_decoration_data: null,
+      banner_color: null,
+      clan: null
+    },
+    attachments: [
+      {
+        id: "1227338263132635217",
+        filename: "WOW.jpg",
+        size: 55508,
+        url: "https://cdn.discordapp.com/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+        proxy_url: "https://media.discordapp.net/attachments/367936426962190336/1227338263132635217/WOW.jpg?ex=66280ad4&is=661595d4&hm=023a7a708935cf1c0cab01085486b939e1c08f92789f7a9b5c0c8ac5914c401c&",
+        width: 1080,
+        height: 1022,
+        content_type: "image/jpeg",
+        content_scan_version: 1,
+        placeholder: "YPcFD4LU/YlNhpxWh4iLiJhn+iRpT8AH",
+        placeholder_version: 1
+      }
+    ]
+  },
+  memeId: "7456180d-fa94-4465-bcb6-ea80524c67e8",
+  vision: {
+    textAnnotations: [],
+    localizedObjectAnnotations: [],
+    safeSearchAnnotation: null,
+    imagePropertiesAnnotation: null,
+    error: null,
+    cropHintsAnnotation: null,
+    fullTextAnnotation: {
+      text: "damn thats crazy"
+    },
+    webDetection: null,
+    productSearchResults: null,
+    context: null
+  },
+  guild: {
+    id: "239638846688460802",
+    name: "The Philosopher's Meme",
+    icon: "bda0d70d3bb2ec4df2dee343029d990d",
+    description: "An applied philosophical community of practice for creatives, intellectuals, researchers, and developers. https://tpm.ac",
+    home_header: null,
+    splash: "0d80f531b01ab5abc8d153386f2af3e6",
+    discovery_splash: "b35170de857b5eb58b1065131afa0826",
+    banner: null,
+    owner_id: "103361901899038720",
+    application_id: null,
+    region: "us-south",
+    afk_channel_id: null,
+    afk_timeout: 300,
+    system_channel_id: "934578834534182953",
+    system_channel_flags: 0,
+    widget_enabled: true,
+    widget_channel_id: "301188489490202626",
+    verification_level: 3,
+    default_message_notifications: 1,
+    mfa_level: 1,
+    explicit_content_filter: 2,
+    max_presences: null,
+    max_members: 500000,
+    max_stage_video_channel_users: 50,
+    max_video_channel_users: 25,
+    vanity_url_code: null,
+    premium_tier: 1,
+    premium_subscription_count: 3,
+    preferred_locale: "en-US",
+    rules_channel_id: "301188489490202626",
+    safety_alerts_channel_id: "837815188970537031",
+    public_updates_channel_id: "837815188970537031",
+    hub_type: null,
+    premium_progress_bar_enabled: true,
+    latest_onboarding_question_id: "1136794072313507860",
+    nsfw: false,
+    nsfw_level: 0,
+    incidents_data: null,
+    inventory_settings: null,
+    embed_enabled: true,
+    embed_channel_id: "301188489490202626"
+  },
+  channel: {
+    id: "367936426962190336",
+    type: 0,
+    last_message_id: "1227404638450483340",
+    flags: 0,
+    last_pin_timestamp: "2022-07-12T12:05:47+00:00",
+    guild_id: "239638846688460802",
+    name: "shitposts_memes",
+    parent_id: "1023698856191471667",
+    rate_limit_per_user: 0,
+    topic: "memes, shitposts; memeing, shitposting",
+    position: 38,
+    nsfw: false,
+    icon_emoji: {
+      id: null,
+      name: "🤪"
+    },
+    theme_color: null
+  },
+  createdAt: new Date(),
+  updatedAt: new Date(),
+}