changes for prod container

archiver/Dockerfile

@@ -1,21 +1,7 @@
-FROM node:20 as builder
-WORKDIR /app
-
-COPY . ./
-RUN npm i
-RUN npm run build
-
-# execution
 FROM node:20
-WORKDIR /app
-COPY --from=builder /app/build /app
-
-# packages are not bundled in the build because sharp uses native modules
-COPY package.json /app/package.json
-RUN npm i --include prod
-
-# prisma setup
-COPY ./prisma /app/prisma
+WORKDIR /usr/src/memestream
+COPY . /usr/src/memestream
+RUN npm i
 RUN npx prisma generate
 
-CMD node index.js
+CMD npm start

archiver/build/index.js

@@ -1,5 +1,7 @@
 // index.ts
 import "dotenv/config";
+import { exec as execCb } from "child_process";
+import { promisify } from "util";
 import pino from "pino";
 import * as Minio from "minio";
 import Vision from "@google-cloud/vision";
@@ -302,6 +304,7 @@ async function memeService(baseLog, prisma, meili) {
 BigInt.prototype.toJSON = function bigIntStringify() {
   return this.toString();
 };
+var exec = promisify(execCb);
 async function main() {
   const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default;
   const log = pino({ base: {} });
@@ -331,6 +334,11 @@ async function main() {
     1095060717117374687n
     // aella memes
   ];
+  async function migrateDb() {
+    const output = await exec("prisma migrate deploy");
+    log.info(output);
+  }
+  await migrateDb();
   await Promise.all(CHANNEL_IDS.map((id) => scraper2.processChannel(id)));
 }
 main().then(() => true).catch((e) => console.error(e));

archiver/index.ts

@@ -23,17 +23,19 @@ BigInt.prototype.toJSON = function bigIntStringify() {
 const exec = promisify(execCb)
 
 async function main() {
-    const VISION_CREDENTIALS = (await import(process.env.GOOGLE_APPLICATION_CREDENTIALS, { with: { type: "json" } })).default
-
     const log = pino({ base: {} });
+    const VISION_CREDENTIALS = JSON.parse(process.env.VISION_CREDENTIALS)
 
     const vision = new Vision.ImageAnnotatorClient({ credentials: VISION_CREDENTIALS })
     const prisma = new PrismaClient()
     const minio = new Minio.Client({
-        endPoint: process.env.STORAGE_URL,
+        endPoint: process.env.STORAGE_ENDPOINT,
         accessKey: process.env.STORAGE_ACCESS_KEY,
         secretKey: process.env.STORAGE_SECRET_KEY,
+        useSSL: false,
+        port: 9000,
     })
+
     const meili = new MeiliSearch({
       host: process.env.MEILI_URL,
       apiKey: process.env.MEILI_MASTER_KEY,
@@ -48,7 +50,7 @@ async function main() {
         367936467592413194n,  // tpm screenshots
         1028511952277864501n,  // tpm sludge
         725835129326665768n, // machinegirl memes
-        1095060717117374687n, // aella memes
+        // 1095060717117374687n, // aella memes
     ]
 
     async function migrateDb() {
@@ -58,6 +60,8 @@ async function main() {
 
     await migrateDb()
 
+    log.info('ready')
+
     await Promise.all(CHANNEL_IDS.map(id => scraper.processChannel(id)))
 }
 

archiver/package.json

@@ -5,8 +5,7 @@
   "main": "index.ts",
   "type": "module",
   "scripts": {
-    "start": "esrun index.ts | pino-pretty",
-    "start:bundle": "node index.js",
+    "start": "esrun index.ts",
     "dev": "esrun --watch index.ts | pino-pretty",
     "build": "esbuild index.ts --bundle --format=esm --outdir=build --platform=node --target=node22.1 --packages=external",
     "migrate:make": "prisma migrate dev --name",

archiver/src/memes.ts

@@ -6,11 +6,23 @@ import _ from 'lodash'
 
 import * as attachments from './attachments'
 
+// where to post the imgs to
 const STORAGE_BUCKET = process.env.STORAGE_BUCKET
+// the host the client will fetch the img from
+const STORAGE_HOST = process.env.STORAGE_HOST
+
+// name for the key the client uses
+const MEILI_SEARCH_KEY_UUID = process.env.MEILI_SEARCH_KEY_UUID
+const MEILI_SEARCH_KEY_NAME = process.env.MEILI_SEARCH_KEY_NAME
+const MEILI_INDEX = process.env.MEILI_INDEX || 'memes'
 
 function memeUrl(a: Attachment) {
-    const STORAGE_URL = process.env.STORAGE_URL
-    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `https://${STORAGE_URL}`)
+    const STORAGE_ENDPOINT = process.env.STORAGE_ENDPOINT
+
+    // thisis broken
+    // needs to be actual url
+
+    const u = new URL(`${STORAGE_BUCKET}/${attachments.objectName(a)}`, `${STORAGE_HOST}`)
     return u.href
 }
 
@@ -39,18 +51,37 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
         return m
     }
 
+    async function ensureMeiliSearchKey() {
+        const keys = await meili.getKeys()
+        const searchKey = keys.results.find(k => k.name === MEILI_SEARCH_KEY_NAME)
+        if (searchKey) {
+            log.info(`Public search key=${searchKey.key}`)
+            return searchKey
+        }
+        const created = meili.createKey({
+            name: MEILI_SEARCH_KEY_NAME,
+            actions: ['search', 'documents.get', 'indexes.get', 'stats.get', 'version'],
+            indexes: [MEILI_INDEX],
+            // never expire
+            expiresAt: null,
+            uid: MEILI_SEARCH_KEY_UUID,
+            description: 'Memestream - Public Search Key'
+        })
+        return created
+    }
+
     async function ensureMeiliIndex() {
-        const task = await meili.createIndex('memes', { primaryKey: 'id' })
+        const task = await meili.createIndex(MEILI_INDEX, { primaryKey: 'id' })
         const result = await meili.waitForTask(task.taskUid)
         log.debug(result);
-        const index = await meili.index('memes')
+        const index = await meili.index(MEILI_INDEX)
         await index.updateSortableAttributes(['postedAt'])
         return index;
     }
 
     // BLOWS IT UP
     async function resetMeili() {
-        const delTask = await meili.deleteIndex('memes')
+        const delTask = await meili.deleteIndex(MEILI_INDEX)
         const delResult = await meili.waitForTask(delTask.taskUid)
         const index = await ensureMeiliIndex()
         const memes = (await prisma.meme.findMany())
@@ -64,16 +95,18 @@ export default async function memeService(baseLog: Logger, prisma: PrismaClient,
     }
 
     async function persist(m: Meme) {
-        const index = meili.index('memes')
+        const index = meili.index(MEILI_INDEX)
         const doc = ({ ...m, postedAt: m.postedAt.valueOf() })
         return index.updateDocuments([doc])
     }
 
     await ensureMeiliIndex();
+    await ensureMeiliSearchKey()
 
     // BAM
     // await resetMeili();
 
+
     return {
         ensureMeiliIndex,
         resetMeili,

archiver/src/scraper.ts

@@ -22,7 +22,7 @@ function convertSnowflakeToDate(snowflake) {
     return new Date(Number(milliseconds) + DISCORD_EPOCH)
 }
 
-export default async function scraper(
+export default async function discordScraper(
     discordAuth,
     vision: ImageAnnotatorClient,
     minio: Minio.Client,