Merge branch 'master' of ssh://strix:/home/ntr/repos/gimmeme

.dockerignore

@@ -1,6 +1,16 @@
-data/
-node_modules/
-
-
 .env
+*.env
+.env*
+
+.venv/
 vision-credentials.json
+__pycache__/
+
+node_modules
+
+# drop the workspace file cause it leaks info about personal machines
+*.sublime-workspace
+
+data/
+meili_data/
+data.bak/

.gitignore

@@ -1,4 +1,6 @@
 .env
+*.env
+.env*
 
 .venv/
 vision-credentials.json

archiver/Dockerfile

@@ -1,21 +1,7 @@
-FROM node:20 as builder
-WORKDIR /app
-
-COPY . ./
-RUN npm i
-RUN npm run build
-
-# execution
 FROM node:20
-WORKDIR /app
+WORKDIR /usr/src/memestream
-COPY --from=builder /app/build /app
+COPY . /usr/src/memestream
-
+RUN npm i
-# packages are not bundled in the build because sharp uses native modules
-COPY package.json /app/package.json
-RUN npm i --include prod
-
-# prisma setup
-COPY ./prisma /app/prisma
 RUN npx prisma generate
 
-CMD node index.js
+CMD npm start

archiver/TODO.md

@@ -2,7 +2,4 @@
 
 ## TODO
 
-[ ] Backfill last N images from a channel
-[ ] Set oldest_img_id for channel
-[ ]
 [ ] change downloader to url type

archiver/build/index.js

@@ -1,5 +1,7 @@
 // index.ts
 import "dotenv/config";
+import { exec as execCb } from "child_process";
+import { promisify } from "util";
 import pino from "pino";
 import * as Minio from "minio";
 import Vision from "@google-cloud/vision";
@@ -302,6 +304,7 @@ async function memeService(baseLog, prisma, meili) {
 BigInt.prototype.toJSON = function bigIntStringify() {
   return this.toString();
 };
+var exec = promisify(execCb);
 async function main() {
   const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
   const log = pino({ base: {} });
@@ -331,6 +334,11 @@ async function main() {
     1095060717117374687n
     // aella memes
   ];
+  async function migrateDb() {
+    const output = await exec("prisma migrate deploy");
+    log.info(output);
+  }
+  await migrateDb();
   await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
 }
 main().then(() => true).catch((e) => console.error(e));

archiver/index.ts

@@ -1,6 +1,5 @@
 import "dotenv/config"
 
-import fs from 'fs/promises'
 import { exec as execCb } from 'child_process'
 import { promisify } from 'util'
 
@@ -21,19 +20,21 @@ BigInt.prototype.toJSON = function bigIntStringify() {
 };
 
 const exec = promisify(execCb)
+const log = pino({ base: {} });
 
 async function main() {
-    const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default
+    const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS)
-
-    const log = pino({ base: {} });
 
     const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
     const prisma = new PrismaClient()
     const minio = new Minio.Client({
-        endPoint: process.env.STORAGE_URL,
+        endPoint: process.env.STORAGE_ENDPOINT,
         accessKey: process.env.STORAGE_ACCESS_KEY,
         secretKey: process.env.STORAGE_SECRET_KEY,
+        useSSL: false,
+        port: 9000,
     })
+
     const meili = new MeiliSearch({
       host: process.env.MEILI_URL,
       apiKey: process.env.MEILI_MASTER_KEY,
@@ -41,14 +42,14 @@ async function main() {
 
     const memes = await memeService(log, prisma, meili)
     const attachments = await attachmentService(log, minio, prisma, memes)
-    const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, minio, prisma, log, attachments)
+    const scraper = await discordScraper(process.env.DISCORD_AUTH, vision, prisma, log, attachments)
 
     const CHANNEL_IDS = [
         367936426962190336n,  // TPM shitposts
         367936467592413194n,  // tpm screenshots
         1028511952277864501n,  // tpm sludge
-        725835129326665768n, // machinegirl memes
+        // 725835129326665768n, // machinegirl memes
-        1095060717117374687n, // aella memes
+        // 1095060717117374687n, // aella memes
     ]
 
     async function migrateDb() {
@@ -57,12 +58,9 @@ async function main() {
     }
 
     await migrateDb()
-
     await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
 }
 
 main()
     .then(() => true)
-    .catch(e => console.error(e))
+    .catch(e => log.error(e))
-
-

archiver/package.json

@@ -5,11 +5,12 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "start": "esrun index.ts | pino-pretty",
+    "start": "esrun index.ts",
-    "start:bundle": "node index.js",
+    "util:reset": "esrun reset-meili.ts",
     "dev": "esrun --watch index.ts | pino-pretty",
     "build": "esbuild index.ts --bundle --format=esm --outdir=build --platform=node --target=node22.1 --packages=external",
     "migrate:make": "prisma migrate dev --name",
+    "deploy": "docker build . -t registry.strix.systems/memestream-archiver && docker push registry.strix.systems/memestream-archiver && kubectl rollout restart deployment memestream-archiver",
     "test": "esrun src/scraper.test.ts | pino-pretty"
   },
   "author": "",

archiver/reset-meili.ts

@@ -0,0 +1,30 @@
+import "dotenv/config"
+
+import pino from 'pino'
+
+import { PrismaClient } from '@prisma/client'
+import { MeiliSearch } from 'meilisearch'
+
+import memeService from "./src/memes"
+
+BigInt.prototype.toJSON = function bigIntStringify() {
+    return this.toString();
+};
+
+async function main() {
+    const log = pino({ base: {} });
+    const prisma = new PrismaClient()
+    const meili = new MeiliSearch({
+      host: process.env.MEILI_URL,
+      apiKey: process.env.MEILI_MASTER_KEY,
+    })
+
+    const memes = await memeService(log, prisma, meili)
+    return memes.resetMeili()
+}
+
+main()
+    .then(() => true)
+    .catch(e => console.error(e))
+
+

archiver/src/attachments.ts

@@ -1,12 +1,11 @@
-import sharp from 'sharp'
-import axios from 'axios'
-import mime from 'mime-types'
-
 import _ from 'lodash';
-
+import axios from 'axios'
-import type { Logger } from 'pino'
+import fs from 'fs/promises'
+import mime from 'mime-types'
+import sharp from 'sharp'
 
 import type Minio from 'minio'
+import type { Logger } from 'pino'
 import type { PrismaClient, Attachment, ChannelHistory } from '@prisma/client'
 
 import type { MemeService } from './memes'
@@ -35,12 +34,14 @@ export function objectName(a: Attachment) {
     return `${a.memeId}.${mime.extension(a.contentType)}`
 }
 
-
 export default async function attachmentService(log: Logger, minio: Minio.Client, prisma: PrismaClient, memes: MemeService) {
+    const BUCKET_POLICY = await fs.readFile('./storage-bucket-policy.json', { encoding: 'utf8' })
+
     async function ensureStorageBucketExists() {
         const exists = await minio.bucketExists(STORAGE_BUCKET)
         if (exists) return true
         await minio.makeBucket(STORAGE_BUCKET)
+        await minio.setBucketPolicy(STORAGE_BUCKET, BUCKET_POLICY)
     }
 
     async function storeAttachment(a: Attachment) {

archiver/src/memes.ts

@@ -6,11 +6,23 @@ import _ from 'lodash'
 
 import * as attachments from './attachments'
 
+// where to post the imgs to
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+// the host the client will fetch the img from
+const STORAGE_HOST = process.env.STORAGE_HOST
+
+// name for the key the client uses
+const MEILI_SEARCH_KEY_UUID = process.env.MEILI_SEARCH_KEY_UUID
+const MEILI_SEARCH_KEY_NAME = process.env.MEILI_SEARCH_KEY_NAME
+const MEILI_INDEX = process.env.MEILI_INDEX || 'memes'
 
 function memeUrl(a: Attachment) {
-    const STORAGE_URL = process.env.STORAGE_URL
+    const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT
-    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
+
+    // thisis broken
+    // needs to be actual url
+
+    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `${STORAGE_HOST}`)
     return u.href
 }
 
@@ -39,18 +51,37 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
         return m
     }
 
+    async function ensureMeiliSearchKey() {
+        const keys = await meili.getKeys()
+        const searchKey = keys.results.find(k => k.name === MEILI_SEARCH_KEY_NAME)
+        if (searchKey) {
+            log.info(`Public search key=${searchKey.key}`)
+            return searchKey
+        }
+        const created = meili.createKey({
+            name: MEILI_SEARCH_KEY_NAME,
+            actions: ['search', 'documents.get', 'indexes.get', 'stats.get', 'version'],
+            indexes: [MEILI_INDEX],
+            // never expire
+            expiresAt: null,
+            uid: MEILI_SEARCH_KEY_UUID,
+            description: 'Memestream - Public Search Key'
+        })
+        return created
+    }
+
     async function ensureMeiliIndex() {
-        const task = await meili.createIndex('memes', { primaryKey: 'id' })
+        const task = await meili.createIndex(MEILI_INDEX, { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
-        const index = await meili.index('memes')
+        const index = await meili.index(MEILI_INDEX)
         await index.updateSortableAttributes(['postedAt'])
         return index;
     }
 
     // BLOWS IT UP
     async function resetMeili() {
-        const delTask = await meili.deleteIndex('memes')
+        const delTask = await meili.deleteIndex(MEILI_INDEX)
         const delResult = await meili.waitForTask(delTask.taskUid)
         const index = await ensureMeiliIndex()
         const memes = (await prisma.meme.findMany())
@@ -64,16 +95,18 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     async function persist(m: Meme) {
-        const index = meili.index('memes')
+        const index = meili.index(MEILI_INDEX)
         const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
         return index.updateDocuments([doc])
     }
 
     await ensureMeiliIndex();
+    await ensureMeiliSearchKey()
 
     // BAM
     // await resetMeili();
 
+
     return {
         ensureMeiliIndex,
         resetMeili,

archiver/src/scraper.ts

@@ -8,8 +8,6 @@ import type { ImageAnnotatorClient } from '@google-cloud/vision';
 import type { Logger } from 'pino';
 import type Minio from 'minio'
 
-import sharp from 'sharp';
-
 import type { AttachmentService } from './attachments'
 import { IMAGE_TYPES, VIDEO_TYPES } from './attachments'
 
@@ -22,10 +20,9 @@ function convertSnowflakeToDate(snowflake) {
     return new Date(Number(milliseconds) + DISCORD_EPOCH)
 }
 
-export default async function scraper(
+export default async function discordScraper(
     discordAuth,
     vision: ImageAnnotatorClient,
-    minio: Minio.Client,
     prisma: PrismaClient,
     baseLog: Logger,
     attachments: AttachmentService
@@ -93,6 +90,8 @@ export default async function scraper(
         const guild = await fetchGuildDetails(channel.guild_id)
         const log = baseLog.child({ guild: guild.name, channel: channel.name, channelId })
 
+        log.info("processing...")
+
         async function filterAttachments(messages): Promise<Attachment[]> {
             const acceptedContentTypes = [
                 ...IMAGE_TYPES,
@@ -105,7 +104,7 @@ export default async function scraper(
                 .flatMap(m => m.attachments.map(a => [a, m]))
                 .filter(([a, m]) => acceptedContentTypes.includes(a.content_type));
 
-            log.debug({ validAttachments: validAttachments.length }, 'attachments found in messages')
+            log.info({ validAttachments: validAttachments.length }, 'attachments found in messages')
 
             return Promise.all(validAttachments
                 .map(async ([a, m]) => {
@@ -130,54 +129,62 @@ export default async function scraper(
                     }
 
                     // actually assign vision here
+                    // so we can pass an Attachment to detectText
                     return { ...attachment, vision: await detectText(attachment) }
                 }))
         }
 
-        async function* fetchNew(channelId, log, latestMessageId): AsyncIterable<Attachment> {
+        async function* fetchNew(channelId, log, newMessages): AsyncIterable<Attachment> {
             // reverse the messages because we want to yield them in chronological order
             // the api call returns them most recent first
-            const messages = (await fetchChannelMessagesAfter(channelId, latestMessageId)).reverse()
+            const messages = newMessages.reverse()
             if (messages.length === 0) {
                 log.info('reached latest message')
             }
 
-            log.debug({ messages: messages.length, latestMessageId }, 'fetched new messages')
             yield* await filterAttachments(messages)
         }
 
-        async function* fetchOld(channelId, log, earliestMessageId): AsyncIterable<Attachment> {
+        async function* fetchOld(channelId, log, oldMessages): AsyncIterable<Attachment> {
-            const messages = await fetchChannelMessagesBefore(channelId, earliestMessageId)
+            if (oldMessages.length === 0) {
-            if (messages.length === 0) {
                 log.info('reached beginning of history')
             }
 
-            log.debug({ messages: messages.length, earliestMessageId }, 'fetched older messages')
+            yield* await filterAttachments(oldMessages)
-            yield* await filterAttachments(messages)
         }
 
         const ensureHistory = async () => {
             const stored = await prisma.channelHistory.findFirst({ where: { id: channelId }})
             if (stored) return stored
 
+            const channel = await fetchChannel(channelId)
             const lastMessageId = channel.last_message_id
             return await prisma.channelHistory.create({ data:
                 { latestMessageId: lastMessageId, earliestMessageId: lastMessageId, id: channelId }
             })
         }
 
+        async function processLoop() {
+            // updated by the attachment process
             const history = await ensureHistory()
 
-        async function processLoop() {
+            const newMessages = await fetchChannelMessagesAfter(channelId, history.latestMessageId)
-            const history = await ensureHistory()
+            const newAttachments = fetchNew(channelId, log, newMessages)
-            const newAttachments = fetchNew(channelId, log, history.latestMessageId)
+            const latestMessageId = _.first(newMessages) ? BigInt(_.first(newMessages).id) : 0n
-            const oldAttachments = fetchOld(channelId, log, history.earliestMessageId)
+            if (latestMessageId) await prisma.channelHistory.update({ data: { latestMessageId }, where: { id: history.id }})
+            log.info({ messages: newMessages.length, latestMessageId }, 'fetched new messages')
+
+            const oldMessages = await fetchChannelMessagesBefore(channelId, history.earliestMessageId)
+            const oldAttachments = fetchOld(channelId, log, oldMessages)
+            const earliestMessageId = _.first(oldMessages) ? BigInt(_.first(oldMessages).id) : 0n
+            if (earliestMessageId) await prisma.channelHistory.update({ data: { earliestMessageId }, where: { id: history.id }})
+            log.info({ messages: oldMessages.length, earliestMessageId }, 'fetched older messages')
 
             let oldCount = 0;
             let newCount = 0;
 
             for await (const attachment of newAttachments) {
-                log.debug({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+                log.info({ msg: 'scraped new attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
                 const newHistory = { id: history.id, latestMessageId: attachment.msgId }
                 await attachments.process(attachment, newHistory)
                 // pretty oldschool, the async iterable from the fetch fn does not have a count
@@ -186,7 +193,7 @@ export default async function scraper(
             }
 
             for await (const attachment of oldAttachments) {
-                log.debug({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
+                log.info({ msg: 'scraped old attachment', attachment: _.omit(attachment, ['guild', 'message', 'channel', 'vision']) })
                 const newHistory = { id: history.id, earliestMessageId: attachment.msgId }
                 await attachments.process(attachment, newHistory)
                 newCount++

archiver/storage-bucket-policy.json

@@ -0,0 +1,19 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Resource": [
+                "arn:aws:s3:::memestream/**"
+            ]
+        }
+    ]
+}

client/.dockerignore

@@ -0,0 +1 @@
+node_modules/

client/Dockerfile

@@ -1,6 +1,12 @@
 # build step
 FROM node:20 AS build-env
-WORKDIR /app
+WORKDIR /usr/src/memestream-client
+
+# super annoying
+# ARG REACT_APP_MEILI_ENDPOINT
+# ARG REACT_APP_MEILI_KEY
+# ENV REACT_APP_MEILI_ENDPOINT $REACT_APP_MEILI_ENDPOINT
+# ENV REACT_APP_MEILI_KEY $REACT_APP_MEILI_KEY
 
 COPY . ./
 RUN npm i
@@ -9,5 +15,5 @@ RUN npm run build
 # static server
 FROM nginx
 WORKDIR /srv/app
-COPY --from=build-env /app/build /srv/app
+COPY --from=build-env /usr/src/memestream-client/build /srv/app
 COPY ./nginx.conf /etc/nginx/nginx.conf

client/dev.Dockerfile

@@ -0,0 +1,7 @@
+# build step
+FROM node:20 AS build-env
+WORKDIR /usr/src/memestream-client
+
+COPY . ./
+RUN npm i
+CMD npm start

client/src/App.css

@@ -165,7 +165,7 @@ h1 {
     padding: 0;
   }
 
-  .ais-Hits-list {
+  .ais-Hits-list, .ais-InfiniteHits-list {
     grid-template-columns: 1fr;
   }
 }

client/src/App.js

@@ -11,12 +11,21 @@ import "./App.css";
 import { instantMeiliSearch } from "@meilisearch/instant-meilisearch";
 import { convertSnowflakeToDate } from "./util";
 
+const MEILI_ENDPOINT = process.env.REACT_APP_MEILI_ENDPOINT ||
+  "https://search.memestream.strix.systems"
+
+const MEILI_KEY = process.env.REACT_APP_MEILI_KEY ||
+  '040586799aee83b633a4935d8c0e045e9fde9e78cff0dfe542ee10583272d40d'
+
+console.log({ MEILI_ENDPOINT, MEILI_KEY })
+
 const { searchClient } = instantMeiliSearch(
-  "https://search.memestream.strix.systems",
+  MEILI_ENDPOINT,
-  "ef006ab414a12b49eb797e0637219d5d6bcf3c84b8195c9fac49f2629713705a",
+  MEILI_KEY,
   {
     finitePagination: false,
     placeholderSearch: true,
+    attributesToSnippet: ['text:40'],
   }
 );
 
@@ -32,7 +41,9 @@ const placeholders = [
   'we are not the same',
   'anarchy',
   'goblin',
-  'wise'
+  'wise',
+  'look inside',
+  'gorilla warfare',
 ]
 
 
@@ -61,8 +72,8 @@ const App = () => (
 );
 
 const Hit = ({ hit }) => (
-  <div class="hit" key={hit.id}>
+  <div className="hit" key={hit.id}>
-    <a href={hit.url} class="img" target="_blank" rel="noreferrer">
+    <a href={hit.url} className="img" target="_blank" rel="noreferrer">
       <img src={hit.url} alt={hit.text} />
     </a>
     <div className="info">

docker-compose.build.yaml

@@ -0,0 +1,7 @@
+services:
+  client:
+    image: registry.strix.systems/memestream-client
+    build:
+      context: ./client
+    env_file:
+      - .env.client.prod

docker-compose.yaml

@@ -1,25 +1,39 @@
 ---
 
 services:
+  archiver:
+    image: registry.strix.systems/memestream-archiver
+    command: npm run dev
+    build: ./archiver
+    env_file:
+      - .env.archiver
+    volumes:
+      - './archiver:/usr/src/memestream'
+
+  client:
+    build:
+      context: ./client
+      dockerfile: ./dev.Dockerfile
+    env_file:
+      - .env.client.dev
+    ports:
+      - 3000:3000
+    volumes:
+      - './client:/usr/src/memestream-client'
+
   pg:
     image: postgres:latest
-    environment:
+    env_file:
-      POSTGRES_USER: '${POSTGRES_USER}'
+      - .env.pg
-      POSTGRES_DB: '${POSTGRES_DB}'
-      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
     ports:
       - '5432:5432'
-    volumes:
-      - "./data/pg:/var/lib/postgresql/data"
 
   meili:
     image: getmeili/meilisearch:v1.7
+    env_file:
+      - .env.meili
     ports:
       - '7700:7700'
-    volumes:
-      - './data/meili:/meili_data'
-    environment:
-      MEILI_MASTER_KEY: '${MEILI_MASTER_KEY}'
 
   minio:
     image: quay.io/minio/minio
@@ -27,19 +41,3 @@ services:
       - 9000:9000
       - 9001:9001
     command: ['minio', 'server', '/data', '--console-address', ':9001']
-    volumes:
-      - './data/minio:/data'
-    environment:
-      MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
-      MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
-
-  archiver:
-    build: ./archiver
-    volumes:
-      - './.env:/app/.env'
-      - './archiver/vision-credentials.json:/app/vision-credentials.json'
-
-  client:
-    build: ./client
-    ports:
-      - 8080:8080

src/util.reset.meili.ts

@@ -0,0 +1,6 @@
+import "dotenv/config"
+
+import createServices from "./services"
+const { memes } = await createServices()
+await memes.resetMeili();
+