changes for prod container

.dockerignore

@@ -1,6 +1,16 @@
-data/
-node_modules/
-
-
 .env
+*.env
+.env*
+
+.venv/
 vision-credentials.json
+__pycache__/
+
+node_modules
+
+# drop the workspace file cause it leaks info about personal machines
+*.sublime-workspace
+
+data/
+meili_data/
+data.bak/

.gitignore

@@ -1,4 +1,6 @@
 .env
+*.env
+.env*
 
 .venv/
 vision-credentials.json

archiver/Dockerfile

@@ -1,21 +1,7 @@
-FROM node:20 as builder
-WORKDIR /app
-
-COPY . ./
-RUN npm i
-RUN npm run build
-
-# execution
 FROM node:20
-WORKDIR /app
+WORKDIR /usr/src/memestream
-COPY --from=builder /app/build /app
+COPY . /usr/src/memestream
-
+RUN npm i
-# packages are not bundled in the build because sharp uses native modules
-COPY package.json /app/package.json
-RUN npm i --include prod
-
-# prisma setup
-COPY ./prisma /app/prisma
 RUN npx prisma generate
 
-CMD node index.js
+CMD npm start

archiver/build/index.js

@@ -1,5 +1,7 @@
 // index.ts
 import "dotenv/config";
+import { exec as execCb } from "child_process";
+import { promisify } from "util";
 import pino from "pino";
 import * as Minio from "minio";
 import Vision from "@google-cloud/vision";
@@ -302,6 +304,7 @@ async function memeService(baseLog, prisma, meili) {
 BigInt.prototype.toJSON = function bigIntStringify() {
   return this.toString();
 };
+var exec = promisify(execCb);
 async function main() {
   const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
   const log = pino({ base: {} });
@@ -331,6 +334,11 @@ async function main() {
     1095060717117374687n
     // aella memes
   ];
+  async function migrateDb() {
+    const output = await exec("prisma migrate deploy");
+    log.info(output);
+  }
+  await migrateDb();
   await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
 }
 main().then(() => true).catch((e) => console.error(e));

archiver/index.ts

@@ -23,17 +23,19 @@ BigInt.prototype.toJSON = function bigIntStringify() {
 const exec = promisify(execCb)
 
 async function main() {
-    const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default
-
     const log = pino({ base: {} });
+    const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS)
 
     const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
     const prisma = new PrismaClient()
     const minio = new Minio.Client({
-        endPoint: process.env.STORAGE_URL,
+        endPoint: process.env.STORAGE_ENDPOINT,
         accessKey: process.env.STORAGE_ACCESS_KEY,
         secretKey: process.env.STORAGE_SECRET_KEY,
+        useSSL: false,
+        port: 9000,
     })
+
     const meili = new MeiliSearch({
       host: process.env.MEILI_URL,
       apiKey: process.env.MEILI_MASTER_KEY,
@@ -48,7 +50,7 @@ async function main() {
         367936467592413194n,  // tpm screenshots
         1028511952277864501n,  // tpm sludge
         725835129326665768n, // machinegirl memes
-        1095060717117374687n, // aella memes
+        // 1095060717117374687n, // aella memes
     ]
 
     async function migrateDb() {
@@ -58,6 +60,8 @@ async function main() {
 
     await migrateDb()
 
+    log.info('ready')
+
     await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
 }
 

archiver/package.json

@@ -5,8 +5,7 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "start": "esrun index.ts | pino-pretty",
+    "start": "esrun index.ts",
-    "start:bundle": "node index.js",
     "dev": "esrun --watch index.ts | pino-pretty",
     "build": "esbuild index.ts --bundle --format=esm --outdir=build --platform=node --target=node22.1 --packages=external",
     "migrate:make": "prisma migrate dev --name",

archiver/src/memes.ts

@@ -6,11 +6,23 @@ import _ from 'lodash'
 
 import * as attachments from './attachments'
 
+// where to post the imgs to
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+// the host the client will fetch the img from
+const STORAGE_HOST = process.env.STORAGE_HOST
+
+// name for the key the client uses
+const MEILI_SEARCH_KEY_UUID = process.env.MEILI_SEARCH_KEY_UUID
+const MEILI_SEARCH_KEY_NAME = process.env.MEILI_SEARCH_KEY_NAME
+const MEILI_INDEX = process.env.MEILI_INDEX || 'memes'
 
 function memeUrl(a: Attachment) {
-    const STORAGE_URL = process.env.STORAGE_URL
+    const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT
-    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
+
+    // thisis broken
+    // needs to be actual url
+
+    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `${STORAGE_HOST}`)
     return u.href
 }
 
@@ -39,18 +51,37 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
         return m
     }
 
+    async function ensureMeiliSearchKey() {
+        const keys = await meili.getKeys()
+        const searchKey = keys.results.find(k => k.name === MEILI_SEARCH_KEY_NAME)
+        if (searchKey) {
+            log.info(`Public search key=${searchKey.key}`)
+            return searchKey
+        }
+        const created = meili.createKey({
+            name: MEILI_SEARCH_KEY_NAME,
+            actions: ['search', 'documents.get', 'indexes.get', 'stats.get', 'version'],
+            indexes: [MEILI_INDEX],
+            // never expire
+            expiresAt: null,
+            uid: MEILI_SEARCH_KEY_UUID,
+            description: 'Memestream - Public Search Key'
+        })
+        return created
+    }
+
     async function ensureMeiliIndex() {
-        const task = await meili.createIndex('memes', { primaryKey: 'id' })
+        const task = await meili.createIndex(MEILI_INDEX, { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
-        const index = await meili.index('memes')
+        const index = await meili.index(MEILI_INDEX)
         await index.updateSortableAttributes(['postedAt'])
         return index;
     }
 
     // BLOWS IT UP
     async function resetMeili() {
-        const delTask = await meili.deleteIndex('memes')
+        const delTask = await meili.deleteIndex(MEILI_INDEX)
         const delResult = await meili.waitForTask(delTask.taskUid)
         const index = await ensureMeiliIndex()
         const memes = (await prisma.meme.findMany())
@@ -64,16 +95,18 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     async function persist(m: Meme) {
-        const index = meili.index('memes')
+        const index = meili.index(MEILI_INDEX)
         const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
         return index.updateDocuments([doc])
     }
 
     await ensureMeiliIndex();
+    await ensureMeiliSearchKey()
 
     // BAM
     // await resetMeili();
 
+
     return {
         ensureMeiliIndex,
         resetMeili,

archiver/src/scraper.ts

@@ -22,7 +22,7 @@ function convertSnowflakeToDate(snowflake) {
     return new Date(Number(milliseconds) + DISCORD_EPOCH)
 }
 
-export default async function scraper(
+export default async function discordScraper(
     discordAuth,
     vision: ImageAnnotatorClient,
     minio: Minio.Client,

client/.dockerignore

@@ -0,0 +1 @@
+node_modules/

client/Dockerfile

@@ -1,6 +1,12 @@
 # build step
 FROM node:20 AS build-env
-WORKDIR /app
+WORKDIR /usr/src/memestream-client
+
+# super annoying
+# ARG REACT_APP_MEILI_ENDPOINT
+# ARG REACT_APP_MEILI_KEY
+# ENV REACT_APP_MEILI_ENDPOINT $REACT_APP_MEILI_ENDPOINT
+# ENV REACT_APP_MEILI_KEY $REACT_APP_MEILI_KEY
 
 COPY . ./
 RUN npm i
@@ -9,5 +15,5 @@ RUN npm run build
 # static server
 FROM nginx
 WORKDIR /srv/app
-COPY --from=build-env /app/build /srv/app
+COPY --from=build-env /usr/src/memestream-client/build /srv/app
 COPY ./nginx.conf /etc/nginx/nginx.conf

client/dev.Dockerfile

@@ -0,0 +1,7 @@
+# build step
+FROM node:20 AS build-env
+WORKDIR /usr/src/memestream-client
+
+COPY . ./
+RUN npm i
+CMD npm start

client/src/App.js

@@ -11,9 +11,17 @@ import "./App.css";
 import { instantMeiliSearch } from "@meilisearch/instant-meilisearch";
 import { convertSnowflakeToDate } from "./util";
 
+const MEILI_ENDPOINT = process.env.REACT_APP_MEILI_ENDPOINT ||
+  "https://search.memestream.strix.systems"
+
+const MEILI_KEY = process.env.REACT_APP_MEILI_KEY ||
+  '040586799aee83b633a4935d8c0e045e9fde9e78cff0dfe542ee10583272d40d'
+
+console.log({ MEILI_ENDPOINT, MEILI_KEY })
+
 const { searchClient } = instantMeiliSearch(
-  "https://search.memestream.strix.systems",
+  MEILI_ENDPOINT,
-  "ef006ab414a12b49eb797e0637219d5d6bcf3c84b8195c9fac49f2629713705a",
+  MEILI_KEY,
   {
     finitePagination: false,
     placeholderSearch: true,

docker-compose.build.yaml

@@ -0,0 +1,7 @@
+services:
+  client:
+    image: registry.strix.systems/memestream-client
+    build:
+      context: ./client
+    env_file:
+      - .env.client.prod

docker-compose.yaml

@@ -3,10 +3,8 @@
 services:
   pg:
     image: postgres:latest
-    environment:
+    env_file:
-      POSTGRES_USER: '${POSTGRES_USER}'
+      - .env.pg
-      POSTGRES_DB: '${POSTGRES_DB}'
-      POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
     ports:
       - '5432:5432'
     volumes:
@@ -14,12 +12,12 @@ services:
 
   meili:
     image: getmeili/meilisearch:v1.7
+    env_file:
+      - .env.meili
     ports:
       - '7700:7700'
     volumes:
       - './data/meili:/meili_data'
-    environment:
-      MEILI_MASTER_KEY: '${MEILI_MASTER_KEY}'
 
   minio:
     image: quay.io/minio/minio
@@ -29,17 +27,23 @@ services:
     command: ['minio', 'server', '/data', '--console-address', ':9001']
     volumes:
       - './data/minio:/data'
-    environment:
-      MINIO_ROOT_USER: '${MINIO_ROOT_USER}'
-      MINIO_ROOT_PASSWORD: '${MINIO_ROOT_PASSWORD}'
 
   archiver:
+    image: registry.strix.systems/memestream-archiver
+    command: npm run dev
     build: ./archiver
+    env_file:
+      - .env.archiver
     volumes:
-      - './.env:/app/.env'
+      - './archiver:/usr/src/memestream'
-      - './archiver/vision-credentials.json:/app/vision-credentials.json'
 
   client:
-    build: ./client
+    build:
+      context: ./client
+      dockerfile: ./dev.Dockerfile
+    env_file:
+      - .env.client.dev
     ports:
-      - 8080:8080
+      - 3000:3000
+    volumes:
+      - './client:/usr/src/memestream-client'